From 870b888509ec6fc3c19e41213b369d800aa99f17 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 13 Aug 2025 15:34:16 -0400 Subject: [PATCH 01/69] Bump default: `runner_grace_period=60` Also fix a few inconsistent `runner_initial_grace_period=120` defaults --- .github/workflows/runner.yml | 4 ++-- README.md | 2 +- action.yml | 8 ++++---- src/ec2_gha/start.py | 4 ++-- src/ec2_gha/templates/user-script.sh.templ | 2 +- tests/__snapshots__/test_start.ambr | 6 +++--- tests/test_start.py | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index fb49f5f..52977fa 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -83,10 +83,10 @@ on: required: false type: string runner_grace_period: - description: "Grace period in seconds before terminating instance after last job completes (default 120)" + description: "Grace period in seconds before terminating instance after last job completes (default 60)" required: false type: string - default: "120" + default: "60" runner_initial_grace_period: description: "Grace period in seconds before terminating instance if no jobs start (default 180)" required: false diff --git a/README.md b/README.md index a79d201..a4ac5a6 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): - `ec2_root_device_size` - Root device size in GB (default: 0 = use AMI default) - `ec2_security_group_id` - Security group ID (required for [SSH access], should expose inbound port 22) - `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing) -- `runner_grace_period` - Grace period in seconds before terminating (default: 120) +- `runner_grace_period` - Grace period in seconds before terminating after last job completes (default: 60) - `runner_initial_grace_period` - Grace period in seconds before terminating instance if no jobs start (default: 180) - `ssh_pubkey` - SSH public key (for [SSH access]) diff --git a/action.yml b/action.yml index c0bb7f1..11922db 100644 --- a/action.yml +++ b/action.yml @@ -55,13 +55,13 @@ inputs: description: "The repo to run against. Will use the current repo if not specified." required: false runner_grace_period: - description: "Grace period in seconds before terminating instance after last job completes (default 30)" + description: "Grace period in seconds before terminating instance after last job completes (default 60)" required: false - default: "30" + default: "60" runner_initial_grace_period: - description: "Grace period in seconds before terminating instance if no jobs start (default 120)" + description: "Grace period in seconds before terminating instance if no jobs start (default 180)" required: false - default: "120" + default: "180" runner_registration_timeout: description: "Maximum seconds to wait for runner to register with GitHub (default 300 = 5 minutes)" required: false diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index f8d7b53..ca9fbf1 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -45,7 +45,7 @@ class StartAWS(CreateCloudInstance): runner_initial_grace_period : str Grace period in seconds before terminating if no jobs have started. Defaults to "180". runner_grace_period : str - Grace period in seconds before terminating instance after last job completes. Defaults to "120". + Grace period in seconds before terminating instance after last job completes. Defaults to "60". script : str The script to run on the instance. Defaults to an empty string. security_group_id : str @@ -73,7 +73,7 @@ class StartAWS(CreateCloudInstance): labels: str = "" max_instance_lifetime: str = "360" root_device_size: int = 0 - runner_grace_period: str = "120" + runner_grace_period: str = "60" runner_initial_grace_period: str = "180" runner_release: str = "" script: str = "" diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 006c4e1..94e8294 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -172,7 +172,7 @@ exec >> /tmp/termination-check.log 2>&1 echo "[$$(date)] Checking termination conditions" ACTIVITY_FILE="/var/run/github-runner-last-activity" -GRACE_PERIOD="$${RUNNER_GRACE_PERIOD:-120}" +GRACE_PERIOD="$${RUNNER_GRACE_PERIOD:-60}" INITIAL_GRACE_PERIOD="$${RUNNER_INITIAL_GRACE_PERIOD:-180}" JOB_TRACK_DIR="/var/run/github-runner-jobs" diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index 36cb358..92e1c4a 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -175,7 +175,7 @@ echo "[$(date)] Checking termination conditions" ACTIVITY_FILE="/var/run/github-runner-last-activity" - GRACE_PERIOD="${RUNNER_GRACE_PERIOD:-120}" + GRACE_PERIOD="${RUNNER_GRACE_PERIOD:-60}" INITIAL_GRACE_PERIOD="${RUNNER_INITIAL_GRACE_PERIOD:-180}" JOB_TRACK_DIR="/var/run/github-runner-jobs" @@ -531,7 +531,7 @@ echo "[$(date)] Checking termination conditions" ACTIVITY_FILE="/var/run/github-runner-last-activity" - GRACE_PERIOD="${RUNNER_GRACE_PERIOD:-120}" + GRACE_PERIOD="${RUNNER_GRACE_PERIOD:-60}" INITIAL_GRACE_PERIOD="${RUNNER_INITIAL_GRACE_PERIOD:-180}" JOB_TRACK_DIR="/var/run/github-runner-jobs" @@ -624,7 +624,7 @@ echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=/home/test-user" >> .env - echo "RUNNER_GRACE_PERIOD=30" >> .env + echo "RUNNER_GRACE_PERIOD=60" >> .env # Set up job tracking directory mkdir -p /var/run/github-runner-jobs diff --git a/tests/test_start.py b/tests/test_start.py index d9c1b3f..5ca14e1 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -63,8 +63,8 @@ def test_build_user_data_with_cloudwatch(aws, snapshot): "labels": "test-label", "max_instance_lifetime": "360", "repo": "test-org/test-repo", - "runner_grace_period": "30", - "runner_initial_grace_period": "120", + "runner_grace_period": "60", + "runner_initial_grace_period": "180", "runner_release": "https://example.com/runner.tar.gz", "script": "echo 'test script'", "ssh_pubkey": "", @@ -135,7 +135,7 @@ def test_build_aws_params(complete_params): "labels": "label", "max_instance_lifetime": "360", "repo": "omsf-eco-infra/awsinfratesting", - "runner_grace_period": "120", + "runner_grace_period": "60", "runner_initial_grace_period": "180", "runner_release": "test.tar.gz", "script": "echo 'Hello, World!'", From 87141fe70635b7e67a3ea1d6646eb7f39953971f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 13 Aug 2025 15:36:14 -0400 Subject: [PATCH 02/69] Add `runner_poll_interval`, update README, templ fix --- .github/workflows/runner.yml | 6 +++++ README.md | 27 ++++++++++++++-------- action.yml | 4 ++++ src/ec2_gha/__main__.py | 1 + src/ec2_gha/start.py | 4 ++++ src/ec2_gha/templates/user-script.sh.templ | 6 +++-- tests/__snapshots__/test_start.ambr | 14 +++++++---- tests/test_start.py | 10 +++++--- 8 files changed, 53 insertions(+), 19 deletions(-) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index 52977fa..c7cec54 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -92,6 +92,11 @@ on: required: false type: string default: "180" + runner_poll_interval: + description: "How often (in seconds) to check termination conditions (default 10)" + required: false + type: string + default: "10" runner_registration_timeout: description: "Maximum seconds to wait for runner to register with GitHub (default 300 = 5 minutes)" required: false @@ -155,6 +160,7 @@ jobs: max_instance_lifetime: ${{ inputs.max_instance_lifetime || vars.MAX_INSTANCE_LIFETIME || '360' }} runner_initial_grace_period: ${{ inputs.runner_initial_grace_period }} runner_grace_period: ${{ inputs.runner_grace_period }} + runner_poll_interval: ${{ inputs.runner_poll_interval }} runner_registration_timeout: ${{ inputs.runner_registration_timeout }} ssh_pubkey: ${{ inputs.ssh_pubkey || vars.SSH_PUBKEY }} env: diff --git a/README.md b/README.md index a4ac5a6..be6f684 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): - `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing) - `runner_grace_period` - Grace period in seconds before terminating after last job completes (default: 60) - `runner_initial_grace_period` - Grace period in seconds before terminating instance if no jobs start (default: 180) +- `runner_poll_interval` - How often (in seconds) to check termination conditions (default: 10) - `ssh_pubkey` - SSH public key (for [SSH access]) ## Outputs @@ -146,14 +147,22 @@ jobs: ``` (see also [demo-job-seq], [demo-archs], [demo-matrix-wide]) -### How Termination Works +### Termination logic -1. [GitHub Actions runner hooks][hooks] track job lifecycle events -2. When a job completes, the hook checks if other jobs are running -3. If no jobs are active, a termination check is scheduled after the grace period -4. The instance terminates if still idle when the check runs -5. New jobs starting within the grace period cancel the termination -6. Before shutdown, the runner process is gracefully stopped to remove itself from GitHub +The runner uses [GitHub Actions runner hooks][hooks] to track job start/end events, and a `systemd` timer to poll for when there's: +1. no active jobs running, and +2. no job starts or ends in at least `runner_grace_period` seconds. + +Job start/end events `touch` a "last activity" timestamp file (`/var/run/github-runner-last-activity`), and the systemd timer checks this file every `runner_poll_interval` seconds (default: 10s). + +Each job's status is tracked in a JSON file like `/var/run/github-runner-jobs/.job`. + +The default `runner_grace_period` is 60s, but a longer `runner_initial_grace_period` (default: 180s) is used for the first job after instance boot (to allow time for the runner to register and start). + +When terminating, the runner: +- Gracefully stops the runner process +- Removes itself from GitHub +- Flushes CloudWatch logs ### CloudWatch Logs Integration @@ -237,8 +246,8 @@ Once connected to the instance: - Uses non-ephemeral runners to support instance-reuse across jobs - Uses activity-based termination with systemd timer checks every 30 seconds -- Terminates only after runner_grace_period seconds of inactivity (no race conditions) -- Sets maximum instance lifetime (configurable via `max_instance_lifetime`, default: 6 hours) +- Terminates only after `runner_grace_period` seconds of inactivity (no race conditions) +- Also terminates after `max_instance_lifetime`, as a fail-safe (default: 6 hours) - Supports custom AMIs with pre-installed dependencies ### Default AWS Tags diff --git a/action.yml b/action.yml index 11922db..0d9a9ed 100644 --- a/action.yml +++ b/action.yml @@ -62,6 +62,10 @@ inputs: description: "Grace period in seconds before terminating instance if no jobs start (default 180)" required: false default: "180" + runner_poll_interval: + description: "How often (in seconds) to check termination conditions (default 10)" + required: false + default: "10" runner_registration_timeout: description: "Maximum seconds to wait for runner to register with GitHub (default 300 = 5 minutes)" required: false diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index b88bc2d..bb1f3e8 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -35,6 +35,7 @@ def main(): .update_state("INPUT_MAX_INSTANCE_LIFETIME", "max_instance_lifetime") .update_state("INPUT_RUNNER_GRACE_PERIOD", "runner_grace_period") .update_state("INPUT_RUNNER_INITIAL_GRACE_PERIOD", "runner_initial_grace_period") + .update_state("INPUT_RUNNER_POLL_INTERVAL", "runner_poll_interval") .update_state("INPUT_SSH_PUBKEY", "ssh_pubkey") .update_state("AWS_REGION", "region_name") # default .update_state("INPUT_AWS_REGION", "region_name") # input override diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index ca9fbf1..a5b7cc4 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -46,6 +46,8 @@ class StartAWS(CreateCloudInstance): Grace period in seconds before terminating if no jobs have started. Defaults to "180". runner_grace_period : str Grace period in seconds before terminating instance after last job completes. Defaults to "60". + runner_poll_interval : str + How often (in seconds) to check termination conditions. Defaults to "10". script : str The script to run on the instance. Defaults to an empty string. security_group_id : str @@ -75,6 +77,7 @@ class StartAWS(CreateCloudInstance): root_device_size: int = 0 runner_grace_period: str = "60" runner_initial_grace_period: str = "180" + runner_poll_interval: str = "10" runner_release: str = "" script: str = "" security_group_id: str = "" @@ -276,6 +279,7 @@ def create_instances(self) -> dict[str, str]: "repo": self.repo, "runner_grace_period": self.runner_grace_period, "runner_initial_grace_period": self.runner_initial_grace_period, + "runner_poll_interval": self.runner_poll_interval, "runner_release": self.runner_release, "script": self.script, "ssh_pubkey": self.ssh_pubkey, diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 94e8294..e05c673 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -266,6 +266,8 @@ echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=$homedir" >> .env echo "RUNNER_GRACE_PERIOD=$runner_grace_period" >> .env +echo "RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" >> .env +echo "RUNNER_POLL_INTERVAL=$runner_poll_interval" >> .env # Set up job tracking directory mkdir -p /var/run/github-runner-jobs @@ -284,14 +286,14 @@ Type=oneshot ExecStart=/usr/local/bin/check-runner-termination.sh EOF -cat > /etc/systemd/system/runner-termination-check.timer << 'EOF' +cat > /etc/systemd/system/runner-termination-check.timer << EOF [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service [Timer] OnBootSec=60s -OnUnitActiveSec=30s +OnUnitActiveSec=${runner_poll_interval}s [Install] WantedBy=timers.target diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index 92e1c4a..c844221 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -268,7 +268,9 @@ echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=/home/test-user" >> .env - echo "RUNNER_GRACE_PERIOD=60" >> .env + echo "RUNNER_GRACE_PERIOD=61" >> .env + echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env + echo "RUNNER_POLL_INTERVAL=11" >> .env # Set up job tracking directory mkdir -p /var/run/github-runner-jobs @@ -287,14 +289,14 @@ ExecStart=/usr/local/bin/check-runner-termination.sh EOF - cat > /etc/systemd/system/runner-termination-check.timer << 'EOF' + cat > /etc/systemd/system/runner-termination-check.timer << EOF [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service [Timer] OnBootSec=60s - OnUnitActiveSec=30s + OnUnitActiveSec=11s [Install] WantedBy=timers.target @@ -625,6 +627,8 @@ echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=/home/test-user" >> .env echo "RUNNER_GRACE_PERIOD=60" >> .env + echo "RUNNER_INITIAL_GRACE_PERIOD=180" >> .env + echo "RUNNER_POLL_INTERVAL=10" >> .env # Set up job tracking directory mkdir -p /var/run/github-runner-jobs @@ -643,14 +647,14 @@ ExecStart=/usr/local/bin/check-runner-termination.sh EOF - cat > /etc/systemd/system/runner-termination-check.timer << 'EOF' + cat > /etc/systemd/system/runner-termination-check.timer << EOF [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service [Timer] OnBootSec=60s - OnUnitActiveSec=30s + OnUnitActiveSec=10s [Install] WantedBy=timers.target diff --git a/tests/test_start.py b/tests/test_start.py index 5ca14e1..db21481 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -34,7 +34,9 @@ def test_build_user_data(aws, snapshot): "labels": "test-label", "max_instance_lifetime": "360", "repo": "test-org/test-repo", - "runner_grace_period": "60", + "runner_grace_period": "61", + "runner_initial_grace_period": "181", + "runner_poll_interval": "11", "runner_release": "https://example.com/runner.tar.gz", "script": "echo 'test script'", "ssh_pubkey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host", @@ -65,6 +67,7 @@ def test_build_user_data_with_cloudwatch(aws, snapshot): "repo": "test-org/test-repo", "runner_grace_period": "60", "runner_initial_grace_period": "180", + "runner_poll_interval": "10", "runner_release": "https://example.com/runner.tar.gz", "script": "echo 'test script'", "ssh_pubkey": "", @@ -135,8 +138,9 @@ def test_build_aws_params(complete_params): "labels": "label", "max_instance_lifetime": "360", "repo": "omsf-eco-infra/awsinfratesting", - "runner_grace_period": "60", - "runner_initial_grace_period": "180", + "runner_grace_period": "61", + "runner_initial_grace_period": "181", + "runner_poll_interval": "11", "runner_release": "test.tar.gz", "script": "echo 'Hello, World!'", "ssh_pubkey": "", From 601ef7d8a9b734da9a3dcfaba135ba3878812426 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 13 Aug 2025 15:37:59 -0400 Subject: [PATCH 03/69] Improve Tags --- README.md | 6 ++--- src/ec2_gha/start.py | 56 ++++++++++++++++++++++++++------------------ tests/test_start.py | 6 ++--- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index be6f684..fd4cc82 100644 --- a/README.md +++ b/README.md @@ -254,9 +254,9 @@ Once connected to the instance: The action automatically adds these tags to EC2 instances (unless already provided): - `Name`: Auto-generated from repository/workflow/run-number (e.g., "my-repo/test-workflow/#123") -- `repository`: GitHub repository full name -- `workflow`: Workflow name -- `gha_url`: Direct link to the GitHub Actions run +- `Repository`: GitHub repository full name +- `Workflow`: Workflow name +- `URL`: Direct link to the GitHub Actions run These help with debugging and cost tracking. You can override any of these by providing your own tags with the same keys. diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index a5b7cc4..3fbfee0 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -131,41 +131,51 @@ def _build_aws_params(self, user_data_params: dict) -> dict: repo_basename = os.environ["GITHUB_REPOSITORY"].split("/")[-1] name_parts.append(repo_basename) - # Add workflow name if available - if os.environ.get("GITHUB_WORKFLOW"): - # Try to extract just the filename without extension from workflow ref - workflow_ref = os.environ.get("GITHUB_WORKFLOW_REF", "") - if workflow_ref: - # Extract filename from path like "owner/repo/.github/workflows/test.yml@ref" - import re - match = re.search(r'/([^/@]+)\.(yml|yaml)@', workflow_ref) - if match: - name_parts.append(match.group(1)) - else: - name_parts.append(os.environ["GITHUB_WORKFLOW"]) + # Extract the workflow filename (sans extension) and ref + workflow_ref = os.environ.get("GITHUB_WORKFLOW_REF", "") + if workflow_ref: + # Extract filename from path like "owner/repo/.github/workflows/test.yml@ref" + import re + m = re.search(r'/(?P[^/@]+)\.(yml|yaml)@(?P[^@]+)$', workflow_ref) + if m: + # Clean up the ref - remove "refs/heads/" prefix if present + ref = m['ref'] + if ref.startswith('refs/heads/'): + ref = ref[11:] # Remove "refs/heads/" prefix + elif ref.startswith('refs/tags/'): + ref = ref[10:] # Remove "refs/tags/" prefix + name_parts.append(f"{m['name']}@{ref}") else: - name_parts.append(os.environ["GITHUB_WORKFLOW"]) + name_parts.append("???") + else: + name_parts.append("???") # Add run number if available if os.environ.get("GITHUB_RUN_NUMBER"): - name_parts.append(f"#{os.environ['GITHUB_RUN_NUMBER']}") - - # Create Name tag if we have any parts - if name_parts: + # The # acts as separator, don't add a slash before it + run_number = f"#{os.environ['GITHUB_RUN_NUMBER']}" + # Join existing parts with "/" then append run number directly + if name_parts: + name_value = "/".join(name_parts) + run_number + else: + name_value = run_number + default_tags.append({"Key": "Name", "Value": name_value}) + elif name_parts: + # No run number, just join the parts default_tags.append({"Key": "Name", "Value": "/".join(name_parts)}) # Add repository tag if available - if "repository" not in existing_keys and os.environ.get("GITHUB_REPOSITORY"): - default_tags.append({"Key": "repository", "Value": os.environ["GITHUB_REPOSITORY"]}) + if "Repository" not in existing_keys and os.environ.get("GITHUB_REPOSITORY"): + default_tags.append({"Key": "Repository", "Value": os.environ["GITHUB_REPOSITORY"]}) # Add workflow tag if available - if "workflow" not in existing_keys and os.environ.get("GITHUB_WORKFLOW"): - default_tags.append({"Key": "workflow", "Value": os.environ["GITHUB_WORKFLOW"]}) + if "Workflow" not in existing_keys and os.environ.get("GITHUB_WORKFLOW"): + default_tags.append({"Key": "Workflow", "Value": os.environ["GITHUB_WORKFLOW"]}) # Add run URL tag if available - if "gha_url" not in existing_keys and os.environ.get("GITHUB_SERVER_URL") and os.environ.get("GITHUB_REPOSITORY") and os.environ.get("GITHUB_RUN_ID"): + if "URL" not in existing_keys and os.environ.get("GITHUB_SERVER_URL") and os.environ.get("GITHUB_REPOSITORY") and os.environ.get("GITHUB_RUN_ID"): gha_url = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}" - default_tags.append({"Key": "gha_url", "Value": gha_url}) + default_tags.append({"Key": "URL", "Value": gha_url}) # Combine user tags with default tags all_tags = self.tags + default_tags diff --git a/tests/test_start.py b/tests/test_start.py index db21481..217ca1b 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -166,9 +166,9 @@ def test_build_aws_params(complete_params): "Tags": [ {"Key": "Name", "Value": "test"}, {"Key": "Owner", "Value": "test"}, - {"Key": "repository", "Value": "Open-Athena/ec2-gha"}, - {"Key": "workflow", "Value": "CI"}, - {"Key": "gha_url", "Value": "https://github.com/Open-Athena/ec2-gha/actions/runs/16725250800"}, + {"Key": "Repository", "Value": "Open-Athena/ec2-gha"}, + {"Key": "Workflow", "Value": "CI"}, + {"Key": "URL", "Value": "https://github.com/Open-Athena/ec2-gha/actions/runs/16725250800"}, ], } ] From 0b7eed625e76c075b0f2a521a107ed0a1dde6796 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 13 Aug 2025 15:38:10 -0400 Subject: [PATCH 04/69] Add CLAUDE.md --- CLAUDE.md | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..7989d1f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,141 @@ +# ec2-gha + +**ec2-gha** is a GitHub Action for creating ephemeral, self-hosted GitHub Actions runners on AWS EC2 instances. These runners support GPU workloads, automatically terminate when idle, and can handle multi-job workflows. + +## Common Development Commands + +### Testing +```bash +# Install test dependencies +pip install '.[test]' + +# Run tests matching a pattern +cd tests/ && pytest -v -m 'not slow' +``` + +### Linting +```bash +# Ruff is configured in pyproject.toml +ruff check src/ +ruff format src/ +``` + +## Key Architecture Components + +### GitHub Actions Integration +- **`.github/workflows/runner.yml`**: + - Main entrypoint, reusable workflow callable via external workflows' `job.uses` + - Wraps the `action.yml` composite action + - Outputs an `id` that subsequent jobs can pass to `job.runs-on` +- **`action.yml`**: + - Composite action, wraps `Dockerfile` / `ec2_gha` Python module. + - ≈20 input parameters, including: + - AWS/EC2 configs (instance type, AMI, optional CloudWatch log group, keypair/pubkey for SSH-debugging, etc.) + - GitHub runner configurations (timeouts / poll intervals, labels, etc.) + - Outputs: + - `mapping`, `instances` + - When only one instance/runner is created, also outputs `label` and `instance-id` + +### Core Python Modules +- **`src/ec2_gha/__main__.py`**: Entry point that parses environment variables and initiates runner creation +- **`src/ec2_gha/start.py`**: Contains `StartAWS` class handling EC2 operations, instance lifecycle, and template rendering + +### Template System +- **`src/ec2_gha/templates/user-script.sh.templ`**: Main userdata template using Python's String.Template format, includes inline runner lifecycle hooks + +## Versioning + +`runner.yml` runs `actions/checkout` on this repo but, when called from another workflow, GitHub Actions provides no way to checkout the same ref that this workflow was called at. To avoid this, `inputs.action_ref` gets a default value corresponding to a branch or tag name that each commit (in this repo) is expected to be referenced as (e.g. `v2`). This allows the workflow to check out the correct code without the user needing to explicitly pass an `action_ref` input. + +Patch/minor version tags like `v2.0.0`, `v2.1.0` can be created from the `v2` branch, with the `action_ref` default pinned to those values (similar to syncing a project's `pyproject.toml` version with a Git tag pointing at a given commit). + +`ec2-gha`'s initial release uses a `v2` branch because the upstream `start-aws-gha-runner` has published some `v1*` tags. + +### Usage Example +```yaml +# Caller workflow uses the v2 branch +uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@v2 +# The runner.yml on v2 branch has action_ref default of "v2", so it automatically checks out the correct code +``` + +For complete usage examples, see `.github/workflows/demo*.yml`. + +## Development Guidelines + +### Template Modifications +When modifying the userdata template (`user-script.sh.templ`): +- Use `$variable` or `${variable}` syntax for template substitutions +- Escape literal `$` as `$$` +- Test template rendering in `tests/test_start.py` + +### Environment Variables +The action uses a hierarchical input system: +1. Direct workflow inputs (highest priority) +2. Repository/organization variables (`vars.*`) +3. Default values + +GitHub Actions declares env vars prefixed with `INPUT_` for each input, which `start.py` reads. + +### Error Handling +- Use descriptive error messages that help users understand AWS/GitHub configuration issues +- Always clean up AWS resources on failure (instances, etc.) +- Log important operations to assist debugging + +### Instance Lifecycle Management + +#### Termination Logic +The runner uses a polling-based approach to determine when to terminate: + +1. **Job Tracking**: GitHub runner hooks (`job-started`, `job-completed`) track job lifecycle + - Creates/updates JSON files in `/var/run/github-runner-jobs/` + - Updates "last activity" timestamp at `/var/run/github-runner-last-activity` + +2. **Periodic Polling**: Systemd timer runs `check-runner-termination.sh` every `runner_poll_interval` seconds (default: 10s) + - Checks for running jobs (files with `"status":"running"`) + - If no running jobs, compares idle time against grace period + - Grace periods: + - `runner_initial_grace_period` (default: 180s) - Before first job + - `runner_grace_period` (default: 60s) - Between jobs + +3. **Effective Termination Time**: Due to polling interval, actual termination occurs: + - **Minimum**: grace_period seconds after last activity + - **Maximum**: grace_period + runner_poll_interval seconds + - Example: With 60s grace and 10s poll → terminates 60-70s after last job + +4. **Clean Shutdown Sequence**: + - Stop runner process gracefully (SIGINT) + - Remove runner from GitHub (`config.sh remove`) + - Flush CloudWatch logs + - Execute `shutdown -h now` + +### AWS Resource Tagging +By default, launched EC2 instances are Tagged with: +- `Name`: `f"{repo}/{workflow}#{run_number}"` +- `Repository`: GitHub repository name +- `Workflow`: Workflow name +- `URL`: Direct link to the GitHub Actions run + +## Important Implementation Details + +### Multi-Job Support +- Runners are non-ephemeral to support instance reuse +- Job tracking via GitHub runner hooks (job-started, job-completed) +- Grace period prevents premature termination between sequential jobs + +### Security Considerations +- Never log or expose AWS credentials or GitHub tokens +- Use IAM instance profiles for EC2 API access (not credentials) +- Support OIDC authentication for GitHub Actions + +### CloudWatch Integration +When implementing CloudWatch features: +- Logs are streamed from specific paths defined in userdata template +- Instance profile (separate from launch role) required for CloudWatch API access +- Log group must exist before instance creation +- dpkg lock wait (up to 2 minutes) ensures CloudWatch agent installation succeeds on Ubuntu AMIs where cloud-init or unattended-upgrades may be running + +## Testing Checklist + +Before committing changes: +1. Run tests: `cd tests/ && pytest -v -m 'not slow'` +2. Verify template rendering doesn't break From d071dd0ea9df7aef5353f62434d23f606ef8ae87 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 13 Aug 2025 19:32:53 -0400 Subject: [PATCH 05/69] `instance_name` --- .github/workflows/runner.yml | 6 ++++ action.yml | 4 +++ src/ec2_gha/__main__.py | 1 + src/ec2_gha/start.py | 60 ++++++++++++++++++++---------------- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index c7cec54..a0d6957 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -78,6 +78,11 @@ on: description: "Additional userdata script to run on instance startup (before runner starts)" required: false type: string + instance_name: + description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number" + required: false + type: string + default: "$repo/$name#$run_number" max_instance_lifetime: description: "Maximum instance lifetime in minutes before automatic shutdown (falls back to vars.MAX_INSTANCE_LIFETIME, then 360 = 6 hours)" required: false @@ -157,6 +162,7 @@ jobs: ec2_root_device_size: ${{ inputs.ec2_root_device_size }} ec2_security_group_id: ${{ inputs.ec2_security_group_id || vars.EC2_SECURITY_GROUP_ID }} ec2_userdata: ${{ inputs.ec2_userdata }} + instance_name: ${{ inputs.instance_name }} max_instance_lifetime: ${{ inputs.max_instance_lifetime || vars.MAX_INSTANCE_LIFETIME || '360' }} runner_initial_grace_period: ${{ inputs.runner_initial_grace_period }} runner_grace_period: ${{ inputs.runner_grace_period }} diff --git a/action.yml b/action.yml index 0d9a9ed..a2aa38a 100644 --- a/action.yml +++ b/action.yml @@ -47,6 +47,10 @@ inputs: description: "The number of instances to create, defaults to 1" required: true default: "1" + instance_name: + description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number" + required: false + default: "$repo/$name#$run_number" max_instance_lifetime: description: "Maximum instance lifetime in minutes before automatic shutdown (default 360 = 6 hours)" required: false diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index bb1f3e8..edd8f77 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -32,6 +32,7 @@ def main(): .update_state("INPUT_EC2_USERDATA", "userdata") .update_state("INPUT_EXTRA_GH_LABELS", "labels") .update_state("INPUT_INSTANCE_COUNT", "instance_count", type_hint=int) + .update_state("INPUT_INSTANCE_NAME", "instance_name") .update_state("INPUT_MAX_INSTANCE_LIFETIME", "max_instance_lifetime") .update_state("INPUT_RUNNER_GRACE_PERIOD", "runner_grace_period") .update_state("INPUT_RUNNER_INITIAL_GRACE_PERIOD", "runner_initial_grace_period") diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 3fbfee0..7c54add 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -71,6 +71,7 @@ class StartAWS(CreateCloudInstance): cloudwatch_logs_group: str = "" gh_runner_tokens: list[str] = field(default_factory=list) iam_instance_profile: str = "" + instance_name: str = "" key_name: str = "" labels: str = "" max_instance_lifetime: str = "360" @@ -123,46 +124,51 @@ def _build_aws_params(self, user_data_params: dict) -> dict: # Add Name tag if not provided if "Name" not in existing_keys: - # Try to create a sensible default Name tag - name_parts = [] + # Build template variables + template_vars = {} - # Use repository basename if available + # Get repository name (just the basename) if os.environ.get("GITHUB_REPOSITORY"): - repo_basename = os.environ["GITHUB_REPOSITORY"].split("/")[-1] - name_parts.append(repo_basename) + template_vars["repo"] = os.environ["GITHUB_REPOSITORY"].split("/")[-1] + else: + template_vars["repo"] = "unknown" + + # Get workflow full name (e.g., "Test pip install") + template_vars["workflow"] = os.environ.get("GITHUB_WORKFLOW", "unknown") - # Extract the workflow filename (sans extension) and ref + # Get workflow filename stem and ref from GITHUB_WORKFLOW_REF workflow_ref = os.environ.get("GITHUB_WORKFLOW_REF", "") if workflow_ref: - # Extract filename from path like "owner/repo/.github/workflows/test.yml@ref" import re + # Extract filename and ref from path like "owner/repo/.github/workflows/test.yml@ref" m = re.search(r'/(?P[^/@]+)\.(yml|yaml)@(?P[^@]+)$', workflow_ref) if m: - # Clean up the ref - remove "refs/heads/" prefix if present + # Get the workflow filename stem (e.g., "install" from "install.yaml") + template_vars["name"] = m['name'] + + # Clean up the ref - remove "refs/heads/" or "refs/tags/" prefix ref = m['ref'] if ref.startswith('refs/heads/'): - ref = ref[11:] # Remove "refs/heads/" prefix + ref = ref[11:] elif ref.startswith('refs/tags/'): - ref = ref[10:] # Remove "refs/tags/" prefix - name_parts.append(f"{m['name']}@{ref}") + ref = ref[10:] + template_vars["ref"] = ref else: - name_parts.append("???") + template_vars["name"] = "unknown" + template_vars["ref"] = "unknown" else: - name_parts.append("???") - - # Add run number if available - if os.environ.get("GITHUB_RUN_NUMBER"): - # The # acts as separator, don't add a slash before it - run_number = f"#{os.environ['GITHUB_RUN_NUMBER']}" - # Join existing parts with "/" then append run number directly - if name_parts: - name_value = "/".join(name_parts) + run_number - else: - name_value = run_number - default_tags.append({"Key": "Name", "Value": name_value}) - elif name_parts: - # No run number, just join the parts - default_tags.append({"Key": "Name", "Value": "/".join(name_parts)}) + template_vars["name"] = "unknown" + template_vars["ref"] = "unknown" + + # Get run number + template_vars["run_number"] = os.environ.get("GITHUB_RUN_NUMBER", "unknown") + + # Apply the instance name template + from string import Template + name_template = Template(self.instance_name) + name_value = name_template.safe_substitute(**template_vars) + + default_tags.append({"Key": "Name", "Value": name_value}) # Add repository tag if available if "Repository" not in existing_keys and os.environ.get("GITHUB_REPOSITORY"): From d8ffe177015f87343755a2cf837adb5b6ca98356 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 14 Aug 2025 08:07:41 -0400 Subject: [PATCH 06/69] `scripts/instance-runtime.py` --- scripts/instance-runtime.py | 615 ++++++++++++++++++++++++++++++++++++ 1 file changed, 615 insertions(+) create mode 100755 scripts/instance-runtime.py diff --git a/scripts/instance-runtime.py b/scripts/instance-runtime.py new file mode 100755 index 0000000..7c26c99 --- /dev/null +++ b/scripts/instance-runtime.py @@ -0,0 +1,615 @@ +#!/usr/bin/env python3 +""" +Analyze EC2 instance runtime and job execution time for GitHub Actions runners. + +Usage: + instance-runtime.py INSTANCE_ID [INSTANCE_ID ...] + instance-runtime.py https://github.com/OWNER/REPO/actions/runs/RUN_ID[/job/JOB_ID] + instance-runtime.py --help +""" + +import argparse +import json +import re +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone, timedelta +from functools import partial +from typing import Dict, List, Optional + +err = partial(print, file=sys.stderr) + +def run_command(cmd: List[str]) -> Optional[str]: + """Run a command and return output, or None on error.""" + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + err(f"Error running command: {' '.join(cmd)}") + err(f"Error: {e.stderr}") + return None + + + + + + +def get_log_streams(instance_id: str, log_group: str = "/aws/ec2/github-runners") -> List[Dict]: + """Get CloudWatch log streams for an instance.""" + cmd = [ + "aws", "logs", "describe-log-streams", + "--log-group-name", log_group, + "--log-stream-name-prefix", instance_id, + "--query", "logStreams", + "--output", "json" + ] + output = run_command(cmd) + if output: + try: + return json.loads(output) + except json.JSONDecodeError: + return [] + return [] + + +def get_log_events(log_group: str, log_stream: str, limit: int = 100, start_from_head: bool = False) -> List[Dict]: + """Get events from a CloudWatch log stream.""" + cmd = [ + "aws", "logs", "get-log-events", + "--log-group-name", log_group, + "--log-stream-name", log_stream, + "--limit", str(limit), + "--output", "json" + ] + if start_from_head: + cmd.append("--start-from-head") + + output = run_command(cmd) + if output: + try: + result = json.loads(output) + return result.get("events", []) + except json.JSONDecodeError: + return [] + return [] + + +def parse_timestamp(ts_str: str) -> Optional[datetime]: + """Parse various timestamp formats.""" + # Try ISO format first + try: + return datetime.fromisoformat(ts_str.replace("+00:00", "+00:00")) + except: + pass + + # Try log format: "Thu Aug 14 00:29:25 UTC 2025" + try: + return datetime.strptime(ts_str, "%a %b %d %H:%M:%S %Z %Y").replace(tzinfo=timezone.utc) + except: + pass + + return None + + +def extract_timestamp_from_log(message: str) -> Optional[datetime]: + """Extract timestamp from log message.""" + # Pattern: [Thu Aug 14 00:29:25 UTC 2025] + match = re.search(r'\[([^]]+UTC \d{4})]', message) + if match: + return parse_timestamp(match.group(1)) + return None + + +def analyze_instance(instance_id: str, log_group: str = "/aws/ec2/github-runners") -> Dict: + """Analyze runtime and job execution for an instance.""" + result = { + "instance_id": instance_id, + "launch_time": None, + "termination_time": None, + "total_runtime_seconds": 0, + "job_runtime_seconds": 0, + "jobs": [], + "state": "unknown", + "instance_type": "unknown", + "tags": {} + } + + + # Get CloudWatch logs + log_streams = get_log_streams(instance_id, log_group) + + # Check if logs are empty (all streams have 0 bytes) + logs_empty = all(stream.get("storedBytes", 0) == 0 for stream in log_streams) if log_streams else True + + # Extract instance info from logs + if log_streams and not logs_empty: + # Try to get launch time and instance type from runner-setup log + for stream in log_streams: + if "/runner-setup" in stream["logStreamName"]: + # Get first events for launch time + events = get_log_events(log_group, stream["logStreamName"], limit=50, start_from_head=True) + + # Get the first timestamp from any log entry as approximate launch time + if not result["launch_time"] and events: + for event in events: + ts = extract_timestamp_from_log(event.get("message", "")) + if ts: + result["launch_time"] = ts + break + + # Look for instance type and other metadata + for event in events: + msg = event.get("message", "") + + # Look for instance type in metadata + if result["instance_type"] == "unknown": + # Try various patterns for instance types + patterns = [ + r'Instance type:\s+(\S+)', + r'instance-type["\s:]+([a-z0-9]+\.[a-z0-9]+)', + r'EC2_INSTANCE_TYPE=([a-z0-9]+\.[a-z0-9]+)', + r'"instance_type":\s*"([a-z0-9]+\.[a-z0-9]+)"', + # Common instance type patterns + r'\b(g4dn\.\w+|g5\.\w+|g5g\.\w+|t[234]\.\w+|t[34][ag]\.\w+|p[234]\.\w+|p4d\.\w+|c[456]\.\w+|c[56]a\.\w+|m[456]\.\w+|m[56]a\.\w+|r[456]\.\w+)\b', + ] + for pattern in patterns: + match = re.search(pattern, msg, re.IGNORECASE) + if match: + result["instance_type"] = match.group(1).lower() + break + + # Look for repository name + if "Repository:" in msg or "GITHUB_REPOSITORY" in msg: + match = re.search(r'Repository:\s+(\S+)|GITHUB_REPOSITORY=(\S+)', msg) + if match: + repo = match.group(1) or match.group(2) + result["tags"]["Repository"] = repo + + # If still no launch time, use the log stream creation time + if not result["launch_time"] and stream.get("creationTime"): + # CloudWatch timestamps are in milliseconds + result["launch_time"] = datetime.fromtimestamp(stream["creationTime"] / 1000, tz=timezone.utc) + + # Find termination time + for stream in log_streams: + if "/termination" in stream["logStreamName"]: + events = get_log_events(log_group, stream["logStreamName"]) + for event in events: + if "proceeding with termination" in event["message"]: + ts = extract_timestamp_from_log(event["message"]) + if ts: + result["termination_time"] = ts + elif "Runner removed from GitHub successfully" in event["message"]: + ts = extract_timestamp_from_log(event["message"]) + if ts and not result["termination_time"]: + result["termination_time"] = ts + + # Determine state based on termination time + if result["termination_time"]: + result["state"] = "terminated" + elif result["launch_time"]: + # If we have launch time but no termination, assume still running + result["state"] = "running" + result["termination_time"] = datetime.now(timezone.utc) + result["still_running"] = True + + # Calculate total runtime + if result["launch_time"] and result["termination_time"]: + delta = result["termination_time"] - result["launch_time"] + result["total_runtime_seconds"] = int(delta.total_seconds()) + + # Analyze job execution times + job_starts = {} + job_ends = {} + + for stream in log_streams: + if "/job-started" in stream["logStreamName"]: + events = get_log_events(log_group, stream["logStreamName"]) + for event in events: + msg = event.get("message", "") + # Parse job start events - look for "Job STARTED" pattern + if "Job STARTED" in msg: + # Extract timestamp + ts = extract_timestamp_from_log(msg) + if ts: + # Extract job name and run info + # Pattern: "Job STARTED : Test pip install - multiple versions/install (Run: 16952719799/11, Attempt: 1)" + match = re.search(r'Job STARTED\s*:\s*([^(]+)\s*\(Run:\s*(\d+)/(\d+)', msg) + if match: + job_name = match.group(1).strip() + run_id = match.group(2) + job_num = match.group(3) + job_key = f"{run_id}/{job_num}" + job_starts[job_key] = (ts, job_name) + + elif "/job-completed" in stream["logStreamName"]: + events = get_log_events(log_group, stream["logStreamName"]) + for event in events: + msg = event.get("message", "") + # Parse job completion events - look for "Job COMPLETED" pattern + if "Job COMPLETED" in msg: + # Extract timestamp + ts = extract_timestamp_from_log(msg) + if ts: + # Extract job name and run info + # Pattern: "Job COMPLETED: Test pip install - multiple versions/install (Run: 16952719799/11, Attempt: 1)" + match = re.search(r'Job COMPLETED\s*:\s*([^(]+)\s*\(Run:\s*(\d+)/(\d+)', msg) + if match: + job_name = match.group(1).strip() + run_id = match.group(2) + job_num = match.group(3) + job_key = f"{run_id}/{job_num}" + job_ends[job_key] = (ts, job_name) + + # Match starts and ends + total_job_time = 0 + for job_key in job_starts: + if job_key in job_ends: + start_ts, start_name = job_starts[job_key] + end_ts, end_name = job_ends[job_key] + duration = int((end_ts - start_ts).total_seconds()) + total_job_time += duration + result["jobs"].append({ + "name": start_name or end_name or job_key, + "start": start_ts.isoformat(), + "end": end_ts.isoformat(), + "duration_seconds": duration + }) + + result["job_runtime_seconds"] = total_job_time + + return result + + +def get_instances_from_github_url(url: str) -> List[str]: + """Extract instance IDs from a GitHub Actions URL.""" + # Parse the URL + match = re.match(r'https://github\.com/([^/]+)/([^/]+)/actions/runs/(\d+)(?:/job/(\d+))?', url) + if not match: + err(f"Error: Invalid GitHub Actions URL format: {url}") + return [] + + owner, repo, run_id, job_id = match.groups() + + # Get jobs for this run + cmd = ["gh", "api", f"repos/{owner}/{repo}/actions/runs/{run_id}/jobs"] + output = run_command(cmd) + if not output: + return [] + + try: + jobs_data = json.loads(output) + except json.JSONDecodeError: + err(f"Error: Could not parse GitHub API response") + return [] + + instance_ids = [] + jobs = jobs_data.get("jobs", []) + + for job in jobs: + # If specific job_id provided, filter to that job + if job_id and str(job.get("id")) != job_id: + continue + + # Look for instance ID in runner name (format: i-xxxxx) + runner_name = job.get("runner_name", "") + match = re.search(r'(i-[0-9a-f]+)', runner_name) + if match: + instance_ids.append(match.group(1)) + + # Also check labels + for label in job.get("labels", []): + match = re.search(r'(i-[0-9a-f]+)', label) + if match: + instance_ids.append(match.group(1)) + + return list(set(instance_ids)) # Remove duplicates + + +def format_duration(seconds: int) -> str: + """Format duration in human-readable format.""" + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + secs = seconds % 60 + + if hours > 0: + return f"{hours}h {minutes}m {secs}s" + elif minutes > 0: + return f"{minutes}m {secs}s" + else: + return f"{secs}s" + + +# Cache for instance prices to avoid repeated API calls +_price_cache = {} +_pricing_api_warned = False + +def get_instance_price(instance_type: str, region: str = "us-east-1") -> float: + """Get the current on-demand price for an instance type.""" + global _pricing_api_warned + + # Check cache first + cache_key = f"{instance_type}:{region}" + if cache_key in _price_cache: + return _price_cache[cache_key] + + # Try AWS Pricing API (only works from us-east-1 region) + # Note: This requires the pricing:GetProducts permission + try: + cmd = [ + "aws", "pricing", "get-products", + "--service-code", "AmazonEC2", + "--region", "us-east-1", # Pricing API only works in us-east-1 + "--filters", + f"Type=TERM_MATCH,Field=instanceType,Value={instance_type}", + f"Type=TERM_MATCH,Field=location,Value={get_region_name(region)}", + "Type=TERM_MATCH,Field=operatingSystem,Value=Linux", + "Type=TERM_MATCH,Field=tenancy,Value=Shared", + "Type=TERM_MATCH,Field=preInstalledSw,Value=NA", + "--max-items", "1", + "--output", "json" + ] + + output = run_command(cmd) + if output and "PriceList" in output: + data = json.loads(output) + price_list = data.get("PriceList", []) + if price_list: + price_data = json.loads(price_list[0]) + on_demand = price_data.get("terms", {}).get("OnDemand", {}) + for term in on_demand.values(): + for price_dimension in term.get("priceDimensions", {}).values(): + price_per_unit = price_dimension.get("pricePerUnit", {}).get("USD") + if price_per_unit: + price = float(price_per_unit) + _price_cache[cache_key] = price + err(f"Got live price for {instance_type} in {region}: ${price:.4f}/hour") + return price + except Exception as e: + # Pricing API might not be available or have permissions + if not _pricing_api_warned: + err(f"Note: Could not fetch live pricing (AWS Pricing API unavailable or no permissions)") + _pricing_api_warned = True + + # No pricing available + _price_cache[cache_key] = 0 + return 0 + + +def get_region_name(region_code: str) -> str: + """Convert region code to region name for pricing API. + + Uses AWS SSM to get the actual region name, falls back to a formatted guess. + """ + # Try to get from AWS SSM parameters (these are publicly available) + try: + cmd = [ + "aws", "ssm", "get-parameter", + "--name", f"/aws/service/global-infrastructure/regions/{region_code}/longName", + "--query", "Parameter.Value", + "--output", "text", + "--region", region_code + ] + output = run_command(cmd) + if output: + return output + except: + pass + + # Fallback: format the region code into a readable name + # us-east-1 -> US East 1, eu-west-2 -> EU West 2, etc. + parts = region_code.split('-') + if len(parts) >= 3: + region_map = { + "us": "US", + "eu": "EU", + "ap": "Asia Pacific", + "ca": "Canada", + "sa": "South America", + "me": "Middle East", + "af": "Africa" + } + area = region_map.get(parts[0], parts[0].upper()) + direction = parts[1].capitalize() + number = parts[2] + return f"{area} ({direction} {number})" + + # Last resort: just return the code + return region_code + + +def calculate_cost(instance_type: str, runtime_seconds: int, region: str = "us-east-1") -> float: + """Calculate cost based on instance type and runtime.""" + hourly_cost = get_instance_price(instance_type, region) + if hourly_cost == 0: + return 0 + + hours = runtime_seconds / 3600 + return hourly_cost * hours + + +def main(): + parser = argparse.ArgumentParser( + description="Analyze EC2 instance runtime and job execution time for GitHub Actions runners.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s i-0abc123def456789 + %(prog)s i-0abc123def456789 i-0def456abc789012 + %(prog)s https://github.com/owner/repo/actions/runs/123456789 + %(prog)s https://github.com/owner/repo/actions/runs/123456789/job/987654321 + %(prog)s --log-group /custom/log/group i-0abc123def456789 + """ + ) + + parser.add_argument( + "targets", + nargs="+", + help="Instance IDs or GitHub Actions URL" + ) + + parser.add_argument( + "--log-group", + default="/aws/ec2/github-runners", + help="CloudWatch Logs group name (default: /aws/ec2/github-runners)" + ) + + parser.add_argument( + "--region", + default="us-east-1", + help="AWS region for pricing (default: us-east-1)" + ) + + parser.add_argument( + "--json", + action="store_true", + help="Output results as JSON" + ) + + parser.add_argument( + "--parallel", + type=int, + default=10, + help="Maximum number of parallel instance lookups (default: 10, use 1 for sequential)" + ) + + args = parser.parse_args() + + # Collect all instance IDs + instance_ids = [] + for target in args.targets: + if target.startswith("https://github.com/"): + ids = get_instances_from_github_url(target) + if ids: + err(f"Found instances from GitHub URL: {', '.join(ids)}") + instance_ids.extend(ids) + else: + err(f"Warning: No instances found for URL: {target}") + elif target.startswith("i-"): + instance_ids.append(target) + else: + err(f"Warning: Skipping invalid target: {target}") + + if not instance_ids: + err("Error: No valid instance IDs found") + sys.exit(1) + + # Analyze instances in parallel + results = [] + total_runtime = 0 + total_job_runtime = 0 + total_cost = 0 + + # Determine parallel execution mode + max_workers = min(args.parallel, len(instance_ids)) + if max_workers > 1: + err(f"Analyzing {len(instance_ids)} instance(s) with {max_workers} parallel workers...") + else: + err(f"Analyzing {len(instance_ids)} instance(s) sequentially...") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all tasks + future_to_instance = { + executor.submit(analyze_instance, instance_id, args.log_group): instance_id + for instance_id in instance_ids + } + + # Process results as they complete + for future in as_completed(future_to_instance): + instance_id = future_to_instance[future] + try: + result = future.result(timeout=30) # 30 second timeout per instance + + # Calculate cost + cost = calculate_cost(result["instance_type"], result["total_runtime_seconds"], args.region) + result["estimated_cost"] = cost + + # Add to results + results.append(result) + total_runtime += result["total_runtime_seconds"] + total_job_runtime += result["job_runtime_seconds"] + total_cost += cost + + except Exception as e: + err(f"Error analyzing {instance_id}: {e}") + # Add failed result + results.append({ + "instance_id": instance_id, + "error": str(e), + "total_runtime_seconds": 0, + "job_runtime_seconds": 0, + "estimated_cost": 0 + }) + + # Sort results by instance ID for consistent output + results.sort(key=lambda x: x.get("instance_id", "")) + + if args.json: + # JSON output + output = { + "instances": results, + "summary": { + "total_instances": len(results), + "total_runtime_seconds": total_runtime, + "total_job_runtime_seconds": total_job_runtime, + "total_idle_seconds": total_runtime - total_job_runtime, + "estimated_total_cost": round(total_cost, 4) + } + } + print(json.dumps(output, indent=2, default=str)) + else: + # Human-readable output + print("\n" + "="*80) + for result in results: + print(f"\nInstance: {result['instance_id']}") + print(f" Type: {result['instance_type']}") + print(f" State: {result['state']}") + + if result.get("tags", {}).get("Name"): + print(f" Name: {result['tags']['Name']}") + + if result["launch_time"]: + print(f" Launch Time: {result['launch_time']}") + + if result["termination_time"]: + if result.get("still_running"): + print(f" Current Time: {result['termination_time']} (still running)") + else: + print(f" Termination Time: {result['termination_time']}") + + print(f" Total Runtime: {format_duration(result['total_runtime_seconds'])} ({result['total_runtime_seconds']}s)") + print(f" Job Runtime: {format_duration(result['job_runtime_seconds'])} ({result['job_runtime_seconds']}s)") + + idle_time = result['total_runtime_seconds'] - result['job_runtime_seconds'] + print(f" Idle Time: {format_duration(idle_time)} ({idle_time}s)") + + if result['total_runtime_seconds'] > 0: + utilization = (result['job_runtime_seconds'] / result['total_runtime_seconds']) * 100 + print(f" Utilization: {utilization:.1f}%") + + if result.get("estimated_cost", 0) > 0: + print(f" Estimated Cost: ${result['estimated_cost']:.4f}") + + if result["jobs"]: + print(f" Jobs ({len(result['jobs'])}):") + for job in result["jobs"]: + print(f" - {job['name']}: {format_duration(job['duration_seconds'])}") + + print("\n" + "="*80) + print("SUMMARY") + print(f" Total Instances: {len(results)}") + print(f" Total Runtime: {format_duration(total_runtime)} ({total_runtime}s)") + print(f" Total Job Runtime: {format_duration(total_job_runtime)} ({total_job_runtime}s)") + print(f" Total Idle Time: {format_duration(total_runtime - total_job_runtime)} ({total_runtime - total_job_runtime}s)") + + if total_runtime > 0: + overall_utilization = (total_job_runtime / total_runtime) * 100 + print(f" Overall Utilization: {overall_utilization:.1f}%") + + if total_cost > 0: + print(f" Estimated Total Cost: ${total_cost:.4f} (from AWS Pricing API)") + + +if __name__ == "__main__": + main() From fc1117396d0fe02b3cf21822b2294f994e1fce04 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 17 Aug 2025 20:43:46 -0400 Subject: [PATCH 07/69] v2 + timer fix --- .github/test-scripts/gpu-benchmark.py | 2 +- .github/workflows/README.md | 75 + .github/workflows/demo-archs.yml | 84 +- .github/workflows/demo-gpu-job-seq.yml | 137 ++ .github/workflows/demo-gpu-minimal.yml | 18 +- .github/workflows/demo-matrix-wide.yml | 2 + .github/workflows/demo-multi-instance.yml | 59 +- .github/workflows/demo-multi-job.yml | 86 +- .github/workflows/demos.yml | 17 +- .github/workflows/runner.yml | 67 +- CLAUDE.md | 11 + README.md | 165 +- action.yml | 49 +- img/demos#25 1.png | Bin 0 -> 186902 bytes img/mamba#12.png | Bin 0 -> 87075 bytes scripts/instance-runtime.py | 139 +- scripts/update-snapshots.sh | 6 + src/ec2_gha/__main__.py | 31 +- src/ec2_gha/defaults.py | 20 + src/ec2_gha/log_constants.py | 24 + src/ec2_gha/start.py | 35 +- src/ec2_gha/templates/user-script.sh.templ | 378 +++-- tests/__snapshots__/test_start.ambr | 1774 ++++++++++++++++---- tests/test_start.py | 246 +-- 24 files changed, 2684 insertions(+), 741 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/demo-gpu-job-seq.yml create mode 100644 img/demos#25 1.png create mode 100644 img/mamba#12.png create mode 100755 scripts/update-snapshots.sh create mode 100644 src/ec2_gha/defaults.py create mode 100644 src/ec2_gha/log_constants.py diff --git a/.github/test-scripts/gpu-benchmark.py b/.github/test-scripts/gpu-benchmark.py index f89a440..3805291 100755 --- a/.github/test-scripts/gpu-benchmark.py +++ b/.github/test-scripts/gpu-benchmark.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """GPU benchmark script for testing PyTorch CUDA capabilities. -Used by ``demo-gpu-dbg.yml``. +Used by ``demo-gpu.yml``. """ import sys diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..3e0024a --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,75 @@ +# `ec2-gha` Demos +This directory contains the reusable workflow and demo workflows for ec2-gha, demonstrating various capabilities. + +For documentation about the main workflow, [`runner.yml`](runner.yml), see [the main README](../../README.md). + + +- [`demos` – run all demo workflows](#demos) +- [GPU demos](#gpu) + - [`gpu-minimal` – `nvidia-smi` "hello world"](#gpu-minimal) + - [`gpu-job-seq` – GPU train/test/eval (sequential jobs)](#gpu-job-seq) + - [Real-world example: Mamba installation testing](#mamba) +- [Architecture & Parallelization](#arch) + - [`archs` – launch x86 and ARM nodes](#archs) + - [`multi-instance` – launch multiple instances, use in matrix](#multi-instance) + - [`multi-job` – launch multiple instances, use individually](#multi-job) + + +## [`demos`](demos.yml) – run all demo workflows +Useful regression test, demonstrates and verifies features. + +[![](../../img/demos%2325%201.png)][demos#25] + +## GPU demos + +### [`gpu-minimal`](demo-gpu-minimal.yml) – `nvidia-smi` "hello world" +- **Instance type:** `g4dn.xlarge` + +### [`gpu-job-seq`](demo-gpu-job-seq.yml) – GPU train/test/eval (sequential jobs) +- Runs 3 jobs sequentially on the same GPU instance (prepare, train, evaluate) +- Uses pre-installed PyTorch from Deep Learning AMI's conda environment +- Runs GPU benchmark with matrix operations and training simulation +- Verifies same GPU is used across all jobs +- Demonstrates instance reuse for multi-stage ML workflows +- **Instance type:** `g4dn.xlarge` +- **Use case:** ML/AI workflow testing with GPU acceleration + +### Real-world example: [Mamba installation testing](https://github.com/Open-Athena/mamba/blob/gha/.github/workflows/install.yaml) +- Tests different versions of `mamba_ssm` package on GPU instances +- **Customizes `instance_name`**: `"$repo/$name==${{ inputs.mamba_version }} (#$run_number)"` + - Results in descriptive names like `"Open-Athena/mamba/install==2.2.5 (#123)"` + - Makes it easy to identify which version is being tested on each instance +- Uses pre-installed PyTorch from DLAMI conda environment +- **Use case:** Package compatibility testing across versions + +[![](../../img/mamba%2312.png)][mamba#12] + +## Architecture & Parallelization + +### [`archs`](demo-archs.yml) – launch x86 and ARM nodes +- Verify architecture-specific behavior +- **Instance types:** `t3.medium` (x86), `t4g.medium` (ARM) + +### [`multi-instance`](demo-multi-instance.yml) – launch multiple instances, use in matrix +- Creates configurable number of instances (default: 3) +- Uses matrix strategy to run jobs in parallel +- Each job runs on its own EC2 instance +- **Customizes `instance_name`**: `"$repo/$name-$idx (#$run_number)"` + - Uses `$idx` template variable (0-based index) to distinguish instances + - Results in names like `"ec2-gha/multi-instance-0 (#123)"`, `"ec2-gha/multi-instance-1 (#123)"`, etc. +- **Instance type:** `t3.medium` +- **Use case:** Parallel test execution + +### [`multi-job`](demo-multi-job.yml) – launch multiple instances, use individually +- Launch 2 instances +- Run build job on first instance +- Run test job on second instance +- Aggregate results from both instances +- **Customizes `instance_name`**: `"$repo/$name-$idx (#$run_number)"` + - Results in names like `"ec2-gha/multi-job-0 (#123)"` and `"ec2-gha/multi-job-1 (#123)"` +- **Instance type:** `t3.medium` +- **Use case:** Pipeline with dedicated instances per stage + +[mamba#12]: https://github.com/Open-Athena/mamba/actions/runs/16972369660/ +[demos#25]: https://github.com/Open-Athena/ec2-gha/actions/runs/17004697889 + diff --git a/.github/workflows/demo-archs.yml b/.github/workflows/demo-archs.yml index 68d3228..857a386 100644 --- a/.github/workflows/demo-archs.yml +++ b/.github/workflows/demo-archs.yml @@ -1,4 +1,4 @@ -name: Demo – 2 GPU instances with different architectures +name: Demo – x86 and ARM instances on: workflow_dispatch: workflow_call: # Tested by `demos.yml` @@ -6,65 +6,73 @@ permissions: id-token: write # Required for AWS OIDC authentication contents: read # Required for actions/checkout jobs: - # Launch EC2 runners for each instance type - g4dn: - name: Launch g4dn + # Launch EC2 runners for each architecture + x86: + name: Launch x86 uses: ./.github/workflows/runner.yml with: - ec2_instance_type: g4dn.xlarge + ec2_instance_type: t3.medium + ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) secrets: inherit - g4ad: - name: Launch g4ad + arm: + name: Launch ARM uses: ./.github/workflows/runner.yml with: - ec2_instance_type: g4ad.xlarge + ec2_instance_type: t4g.medium + ec2_image_id: ami-0aa307ed50ca3e58f # Ubuntu 24.04 LTS arm64 (us-east-1) secrets: inherit # Run jobs directly on launched instances - test-g4dn: - needs: g4dn - if: needs.g4dn.outputs.id != '' - name: Test g4dn - runs-on: ${{ needs.g4dn.outputs.id }} + test-x86: + needs: x86 + if: needs.x86.outputs.id != '' + name: Test x86 + runs-on: ${{ needs.x86.outputs.id }} steps: - - name: GPU test on g4dn.xlarge - run: nvidia-smi # Verify GPU is available - test-g4ad: - needs: g4ad - if: needs.g4ad.outputs.id != '' - name: Test g4ad - runs-on: ${{ needs.g4ad.outputs.id }} + - name: Architecture test on t3.medium + run: | + echo "Architecture: $(uname -m)" + echo "Processor: $(lscpu | grep 'Architecture:' | awk '{print $2}')" + echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" + test-arm: + needs: arm + if: needs.arm.outputs.id != '' + name: Test ARM + runs-on: ${{ needs.arm.outputs.id }} steps: - - name: GPU test on g4ad.xlarge + - name: Architecture test on t4g.medium run: | - lspci | grep -i "vga\|display\|3d\|amd" + echo "Architecture: $(uname -m)" + echo "Processor: $(lscpu | grep 'Architecture:' | awk '{print $2}')" + echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" # Use a `matrix` to run jobs on multiple instances test-matrix: - needs: [g4dn, g4ad] - if: always() && (needs.g4dn.outputs.id != '' || needs.g4ad.outputs.id != '') - name: Test ${{ matrix.instance }} (matrix) + needs: [x86, arm] + if: always() && (needs.x86.outputs.id != '' || needs.arm.outputs.id != '') + name: Test ${{ matrix.arch }} (matrix) continue-on-error: true strategy: matrix: include: - - instance: g4dn - runner: ${{ needs.g4dn.outputs.id }} - - instance: g4ad - runner: ${{ needs.g4ad.outputs.id }} + - arch: x86 + runner: ${{ needs.x86.outputs.id }} + - arch: arm + runner: ${{ needs.arm.outputs.id }} fail-fast: false runs-on: ${{ matrix.runner }} steps: - - name: Matrix GPU test on ${{ matrix.instance }}.xlarge + - name: Matrix architecture test run: | - echo "Running on ${{ matrix.instance }}.xlarge" + echo "Running on ${{ matrix.arch }}" + echo "Architecture: $(uname -m)" echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" - # Check GPU based on instance type - if [[ "${{ matrix.instance }}" == "g4dn" ]]; then - echo "Testing NVIDIA GPU..." - nvidia-smi - elif [[ "${{ matrix.instance }}" == "g4ad" ]]; then - echo "Testing AMD GPU..." - lspci | grep -i "vga\|display\|3d\|amd" + # Verify expected architecture + if [[ "${{ matrix.arch }}" == "x86" ]]; then + echo "Verifying x86_64 architecture..." + [[ "$(uname -m)" == "x86_64" ]] && echo "✓ Confirmed x86_64" || exit 1 + elif [[ "${{ matrix.arch }}" == "arm" ]]; then + echo "Verifying ARM architecture..." + [[ "$(uname -m)" == "aarch64" ]] && echo "✓ Confirmed aarch64" || exit 1 fi diff --git a/.github/workflows/demo-gpu-job-seq.yml b/.github/workflows/demo-gpu-job-seq.yml new file mode 100644 index 0000000..c66a68a --- /dev/null +++ b/.github/workflows/demo-gpu-job-seq.yml @@ -0,0 +1,137 @@ +name: Demo – comprehensive GPU workload with sequential jobs +on: + workflow_dispatch: + inputs: + sleep: + description: "Sleep for this many seconds at the end of each job (optional, for SSH debugging)" + required: false + type: number + default: 0 + ssh_pubkey: + description: "Add this SSH public key to instance's `~/.ssh/authorized_keys` (optional, for debugging)" + required: false + type: string + workflow_call: # Tested by `demos.yml` + inputs: + sleep: + required: false + type: number + default: 0 + ssh_pubkey: + required: false + type: string + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout + +jobs: + ec2: + uses: ./.github/workflows/runner.yml + secrets: inherit + with: + ec2_instance_type: g4dn.xlarge + ec2_image_id: ami-00096836009b16a22 # Deep Learning OSS Nvidia Driver AMI GPU PyTorch + ssh_pubkey: ${{ inputs.ssh_pubkey }} + + # Job 1: Prepare environment and verify GPU + prepare: + needs: ec2 + runs-on: ${{ needs.ec2.outputs.id }} + outputs: + gpu-uuid: ${{ steps.gpu-info.outputs.uuid }} + gpu-name: ${{ steps.gpu-info.outputs.name }} + steps: + - uses: actions/checkout@v4 + + - name: Get GPU info + id: gpu-info + run: | + echo "=== Preparing GPU environment ===" + nvidia-smi + uuid=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader) + name=$(nvidia-smi --query-gpu=name --format=csv,noheader) + echo "uuid=$uuid" >> $GITHUB_OUTPUT + echo "name=$name" >> $GITHUB_OUTPUT + echo "GPU UUID: $uuid" + echo "GPU Name: $name" + + - name: Setup PyTorch environment + run: | + echo "=== Setting up PyTorch environment ===" + # The DLAMI already has PyTorch installed in a conda environment + # Set up environment for GitHub Actions to use the conda env + echo "/opt/conda/envs/pytorch/bin" >> $GITHUB_PATH + echo "CONDA_DEFAULT_ENV=pytorch" >> $GITHUB_ENV + + # Verify PyTorch is available + /opt/conda/envs/pytorch/bin/python -c "import torch; print(f'PyTorch {torch.__version__} with CUDA {torch.version.cuda}')" + echo "PyTorch environment ready" + + - name: Sleep (${{ inputs.sleep }}s) + if: ${{ inputs.sleep > 0 }} + run: | + echo "Sleeping for ${{ inputs.sleep }} seconds..." + echo "Instance IP available in GitHub Actions log for SSH" + sleep ${{ inputs.sleep }} + + # Job 2: Training simulation + train: + needs: [ec2, prepare] + runs-on: ${{ needs.ec2.outputs.id }} + steps: + - uses: actions/checkout@v4 + + - name: Verify same GPU + run: | + echo "=== Training on GPU ===" + current_uuid=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader) + if [[ "$current_uuid" == "${{ needs.prepare.outputs.gpu-uuid }}" ]]; then + echo "✅ Confirmed: Using same GPU as preparation job" + echo "GPU: ${{ needs.prepare.outputs.gpu-name }}" + else + echo "❌ ERROR: Different GPU!" + exit 1 + fi + + - name: Run training benchmark + run: | + echo "=== Running GPU Training Benchmark ===" + # Use the conda environment's Python which has PyTorch pre-installed + /opt/conda/envs/pytorch/bin/python .github/test-scripts/gpu-benchmark.py + + - name: Sleep (${{ inputs.sleep }}s) + if: ${{ inputs.sleep > 0 }} + run: sleep ${{ inputs.sleep }} + + # Job 3: Evaluation/testing + evaluate: + needs: [ec2, train] + runs-on: ${{ needs.ec2.outputs.id }} + steps: + - name: System diagnostics + run: | + echo "=== Final Evaluation ===" + echo "System Info:" + uname -a + lscpu | grep "Model name" || true + free -h + + echo -e "\nGPU Status:" + nvidia-smi --query-gpu=name,memory.total,driver_version,utilization.gpu,temperature.gpu --format=csv + + echo -e "\nDisk Usage:" + df -h / + + echo -e "\nInstance Metadata:" + curl -s -H "X-aws-ec2-metadata-token: $(curl -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 300' http://169.254.169.254/latest/api/token 2>/dev/null)" http://169.254.169.254/latest/meta-data/instance-type || echo "Metadata unavailable" + + - name: Verify multi-job reuse + run: | + echo "=== Verifying EC2 Instance Reuse ===" + echo "This job successfully ran on the same EC2 instance as the previous jobs" + echo "The instance will terminate automatically after idle timeout" + + - name: Sleep (${{ inputs.sleep }}s) + if: ${{ inputs.sleep > 0 }} + run: sleep ${{ inputs.sleep }} diff --git a/.github/workflows/demo-gpu-minimal.yml b/.github/workflows/demo-gpu-minimal.yml index 965fa58..30bf1b8 100644 --- a/.github/workflows/demo-gpu-minimal.yml +++ b/.github/workflows/demo-gpu-minimal.yml @@ -1,8 +1,20 @@ name: Demo – minimal EC2 GPU runner on: workflow_dispatch: + workflow_call: # Tested by `demos.yml` +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: - placeholder: - runs-on: ubuntu-latest + ec2: + # To use from another repo: Open-Athena/ec2-gha/.github/workflows/runner.yml@v2 + uses: ./.github/workflows/runner.yml + secrets: inherit + with: + ec2_instance_type: g4dn.xlarge # ≈$0.56/hr GPU instance: https://instances.vantage.sh/aws/ec2/g4dn.xlarge + ec2_image_id: ami-00096836009b16a22 # Deep Learning OSS Nvidia Driver AMI GPU PyTorch 2.4.1 (Ubuntu 22.04) 20250302 + gpu-test: + needs: ec2 + runs-on: ${{ needs.ec2.outputs.id }} steps: - - run: echo "Placeholder workflow – being developed in #2 / rw/hooks branch" \ No newline at end of file + - run: nvidia-smi # Verify GPU is available diff --git a/.github/workflows/demo-matrix-wide.yml b/.github/workflows/demo-matrix-wide.yml index cd1cff3..e7e3949 100644 --- a/.github/workflows/demo-matrix-wide.yml +++ b/.github/workflows/demo-matrix-wide.yml @@ -9,6 +9,8 @@ jobs: ec2: uses: ./.github/workflows/runner.yml secrets: inherit + with: + ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) test-matrix: needs: ec2 runs-on: ${{ needs.ec2.outputs.id }} diff --git a/.github/workflows/demo-multi-instance.yml b/.github/workflows/demo-multi-instance.yml index 7e21ac2..a2b97f2 100644 --- a/.github/workflows/demo-multi-instance.yml +++ b/.github/workflows/demo-multi-instance.yml @@ -1,8 +1,61 @@ name: Demo – multiple instances for parallel jobs on: workflow_dispatch: + inputs: + instance_count: + description: "Number of EC2 instances to create" + required: false + type: string + default: "3" + workflow_call: # Tested by `demos.yml` + inputs: + instance_count: + required: false + type: string + default: "3" + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout + jobs: - placeholder: - runs-on: ubuntu-latest + ec2: + name: Launch ${{ inputs.instance_count }} EC2 instances + uses: ./.github/workflows/runner.yml + secrets: inherit + with: + instance_name: "$repo/$name-$idx (#$run_number)" + instance_count: ${{ inputs.instance_count }} + ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) + + parallel-jobs: + needs: ec2 + strategy: + matrix: + # Parse the JSON array of runner labels and use them in the matrix + runner: ${{ fromJson(needs.ec2.outputs.instances) }} + runs-on: ${{ matrix.runner }} + name: Job on ${{ matrix.runner }} steps: - - run: echo "Placeholder workflow – being developed in #2 / rw/hooks branch" \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Instance info + run: | + echo "Running on instance with label: ${{ matrix.runner }}" + echo "Hostname: $(hostname)" + echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" + echo "Region: $(curl -s http://169.254.169.254/latest/meta-data/placement/region)" + + - name: Simulate workload + run: | + # Each instance runs independently + DURATION=$((RANDOM % 10 + 5)) + echo "Simulating workload for ${DURATION} seconds..." + sleep $DURATION + echo "Workload complete!" + + - name: Verify parallelism + run: | + echo "Completed at: $(date '+%Y-%m-%d %H:%M:%S')" + echo "This job ran in parallel with other matrix jobs" \ No newline at end of file diff --git a/.github/workflows/demo-multi-job.yml b/.github/workflows/demo-multi-job.yml index 9c584db..d1e0ebb 100644 --- a/.github/workflows/demo-multi-job.yml +++ b/.github/workflows/demo-multi-job.yml @@ -1,8 +1,90 @@ name: Demo – multiple instances for different job types on: workflow_dispatch: + workflow_call: # Tested by `demos.yml` + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout + jobs: - placeholder: + ec2: + name: Launch 2 EC2 instances + uses: ./.github/workflows/runner.yml + secrets: inherit + with: + instance_name: "$repo/$name-$idx (#$run_number)" + instance_count: "2" + ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) + + # First job type - runs on first instance + build-job: + needs: ec2 + runs-on: ${{ fromJson(needs.ec2.outputs.instances)[0] }} + name: Build job + steps: + - uses: actions/checkout@v4 + + - name: Build task + run: | + echo "Running BUILD job on first instance" + echo "Instance: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Simulating build process..." + sleep 10 + echo "Build complete!" + + - name: Create artifact + run: | + echo "Build output from $(hostname)" > build-output.txt + echo "Built at $(date)" >> build-output.txt + + - uses: actions/upload-artifact@v4 + with: + name: build-output + path: build-output.txt + + # Second job type - runs on second instance + test-job: + needs: ec2 + runs-on: ${{ fromJson(needs.ec2.outputs.instances)[1] }} + name: Test job + steps: + - uses: actions/checkout@v4 + + - name: Test task + run: | + echo "Running TEST job on second instance" + echo "Instance: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Simulating test suite..." + sleep 15 + echo "Tests passed!" + + - name: Generate test report + run: | + echo "Test report from $(hostname)" > test-report.txt + echo "Tests run at $(date)" >> test-report.txt + echo "All tests: PASSED" >> test-report.txt + + - uses: actions/upload-artifact@v4 + with: + name: test-report + path: test-report.txt + + # Aggregation job that uses artifacts from both instances + report: + needs: [build-job, test-job] runs-on: ubuntu-latest steps: - - run: echo "Placeholder workflow – being developed in #2 / rw/hooks branch" \ No newline at end of file + - name: Download all artifacts + uses: actions/download-artifact@v4 + + - name: Generate final report + run: | + echo "=== Final Report ===" + echo "Build output:" + cat build-output/build-output.txt + echo "" + echo "Test report:" + cat test-report/test-report.txt + echo "" + echo "Both jobs completed successfully using separate EC2 instances!" \ No newline at end of file diff --git a/.github/workflows/demos.yml b/.github/workflows/demos.yml index 7f5e3de..52dc0a3 100644 --- a/.github/workflows/demos.yml +++ b/.github/workflows/demos.yml @@ -5,14 +5,11 @@ permissions: id-token: write contents: read jobs: - demo-00-minimal: - uses: ./.github/workflows/demo-00-minimal.yml + demo-gpu-minimal: + uses: ./.github/workflows/demo-gpu-minimal.yml secrets: inherit - demo-job-seq: - uses: ./.github/workflows/demo-job-seq.yml - secrets: inherit - demo-gpu-dbg: - uses: ./.github/workflows/demo-gpu-dbg.yml + demo-gpu-job-seq: + uses: ./.github/workflows/demo-gpu-job-seq.yml secrets: inherit demo-archs: uses: ./.github/workflows/demo-archs.yml @@ -20,3 +17,9 @@ jobs: demo-matrix-wide: uses: ./.github/workflows/demo-matrix-wide.yml secrets: inherit + demo-multi-instance: + uses: ./.github/workflows/demo-multi-instance.yml + secrets: inherit + demo-multi-job: + uses: ./.github/workflows/demo-multi-job.yml + secrets: inherit diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index a0d6957..270ad0b 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -26,7 +26,7 @@ on: type: string default: "v2" aws_region: - description: "AWS region for EC2 instances (defaults to us-east-1)" + description: "AWS region for EC2 instances (falls back to vars.AWS_REGION, then us-east-1)" required: false type: string default: "us-east-1" @@ -38,33 +38,30 @@ on: description: "CloudWatch Logs group name for streaming runner logs (leave empty to disable)" required: false type: string - ec2_launch_role: - description: "AWS role ARN to assume for EC2 operations (falls back to vars.EC2_LAUNCH_ROLE; one or the other is required)" + ec2_home_dir: + description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR, then auto-detection)" required: false type: string ec2_image_id: - description: "AWS AMI ID to use (falls back to vars.EC2_IMAGE_ID)" - required: false - type: string - default: "ami-00096836009b16a22" # Deep Learning OSS Nvidia Driver AMI GPU PyTorch - ec2_instance_type: - description: "AWS instance type (falls back to vars.EC2_INSTANCE_TYPE)" + description: "AWS AMI ID to use (required - must be provided via input or vars.EC2_IMAGE_ID)" required: false type: string - default: "g4dn.xlarge" ec2_instance_profile: description: "Instance profile name to attach to launched EC2 instance (required for CloudWatch logging)" required: false type: string - ec2_home_dir: - description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR)" + ec2_instance_type: + description: "AWS instance type (falls back to vars.EC2_INSTANCE_TYPE, then t3.medium)" required: false type: string - default: "/home/ubuntu" ec2_key_name: description: "Name of an EC2 key pair to use for SSH access (falls back to vars.EC2_KEY_NAME)" required: false type: string + ec2_launch_role: + description: "AWS role ARN to assume for EC2 operations (falls back to vars.EC2_LAUNCH_ROLE)" + required: false + type: string ec2_root_device_size: description: "Root device size in GB (0 = use AMI default)" required: false @@ -78,6 +75,11 @@ on: description: "Additional userdata script to run on instance startup (before runner starts)" required: false type: string + instance_count: + description: "Number of EC2 instances to create (for parallel jobs)" + required: false + type: string + default: "1" instance_name: description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number" required: false @@ -88,33 +90,35 @@ on: required: false type: string runner_grace_period: - description: "Grace period in seconds before terminating instance after last job completes (default 60)" + description: "Grace period in seconds before terminating instance after last job completes (falls back to vars.RUNNER_GRACE_PERIOD, then 60)" required: false type: string - default: "60" runner_initial_grace_period: - description: "Grace period in seconds before terminating instance if no jobs start (default 180)" + description: "Grace period in seconds before terminating instance if no jobs start (falls back to vars.RUNNER_INITIAL_GRACE_PERIOD, then 180)" required: false type: string - default: "180" runner_poll_interval: - description: "How often (in seconds) to check termination conditions (default 10)" + description: "How often (in seconds) to check termination conditions (falls back to vars.RUNNER_POLL_INTERVAL, then 10)" required: false type: string - default: "10" runner_registration_timeout: - description: "Maximum seconds to wait for runner to register with GitHub (default 300 = 5 minutes)" + description: "Maximum seconds to wait for runner to register with GitHub (falls back to vars.RUNNER_REGISTRATION_TIMEOUT, then 360 = 6 minutes)" required: false type: string - default: "300" ssh_pubkey: description: "SSH public key to add to authorized_keys (falls back to vars.SSH_PUBKEY)" required: false type: string outputs: id: - description: "Instance ID for runs-on" + description: "Instance ID for runs-on (single instance)" value: ${{ jobs.launch.outputs.id }} + instances: + description: "JSON array of runner labels (for multiple instances)" + value: ${{ jobs.launch.outputs.instances }} + mapping: + description: "JSON object mapping instance IDs to runner labels" + value: ${{ jobs.launch.outputs.mapping }} permissions: id-token: write # Required for AWS OIDC @@ -125,6 +129,8 @@ jobs: runs-on: ubuntu-latest outputs: id: ${{ steps.aws-start.outputs.label }} + instances: ${{ steps.aws-start.outputs.instances }} + mapping: ${{ steps.aws-start.outputs.mapping }} steps: - name: Check EC2_LAUNCH_ROLE configuration run: | @@ -151,23 +157,24 @@ jobs: id: aws-start uses: ./ with: - aws_region: ${{ inputs.aws_region }} + aws_region: ${{ inputs.aws_region || vars.AWS_REGION }} aws_tags: ${{ inputs.aws_tags }} cloudwatch_logs_group: ${{ inputs.cloudwatch_logs_group || vars.CLOUDWATCH_LOGS_GROUP }} + ec2_home_dir: ${{ inputs.ec2_home_dir || vars.EC2_HOME_DIR }} ec2_image_id: ${{ inputs.ec2_image_id || vars.EC2_IMAGE_ID }} - ec2_instance_type: ${{ inputs.ec2_instance_type || vars.EC2_INSTANCE_TYPE }} ec2_instance_profile: ${{ inputs.ec2_instance_profile || vars.EC2_INSTANCE_PROFILE }} - ec2_home_dir: ${{ inputs.ec2_home_dir || vars.EC2_HOME_DIR }} + ec2_instance_type: ${{ inputs.ec2_instance_type || vars.EC2_INSTANCE_TYPE }} ec2_key_name: ${{ inputs.ec2_key_name || vars.EC2_KEY_NAME }} ec2_root_device_size: ${{ inputs.ec2_root_device_size }} ec2_security_group_id: ${{ inputs.ec2_security_group_id || vars.EC2_SECURITY_GROUP_ID }} ec2_userdata: ${{ inputs.ec2_userdata }} + instance_count: ${{ inputs.instance_count }} instance_name: ${{ inputs.instance_name }} - max_instance_lifetime: ${{ inputs.max_instance_lifetime || vars.MAX_INSTANCE_LIFETIME || '360' }} - runner_initial_grace_period: ${{ inputs.runner_initial_grace_period }} - runner_grace_period: ${{ inputs.runner_grace_period }} - runner_poll_interval: ${{ inputs.runner_poll_interval }} - runner_registration_timeout: ${{ inputs.runner_registration_timeout }} + max_instance_lifetime: ${{ inputs.max_instance_lifetime || vars.MAX_INSTANCE_LIFETIME }} + runner_grace_period: ${{ inputs.runner_grace_period || vars.RUNNER_GRACE_PERIOD }} + runner_initial_grace_period: ${{ inputs.runner_initial_grace_period || vars.RUNNER_INITIAL_GRACE_PERIOD }} + runner_poll_interval: ${{ inputs.runner_poll_interval || vars.RUNNER_POLL_INTERVAL }} + runner_registration_timeout: ${{ inputs.runner_registration_timeout || vars.RUNNER_REGISTRATION_TIMEOUT }} ssh_pubkey: ${{ inputs.ssh_pubkey || vars.SSH_PUBKEY }} env: GH_PAT: ${{ secrets.GH_SA_TOKEN }} diff --git a/CLAUDE.md b/CLAUDE.md index 7989d1f..a32201f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,6 +4,9 @@ ## Common Development Commands +- Don't explicitly set the `AWS_PROFILE` (e.g. to `oa-ci-dev`) in your commands; assume it's set for you out of band, verify if you need. +- Instance userdata (rendered form of `src/ec2_gha/templates/user-script.sh.templ`) has to stay under 16KiB. + ### Testing ```bash # Install test dependencies @@ -11,6 +14,14 @@ pip install '.[test]' # Run tests matching a pattern cd tests/ && pytest -v -m 'not slow' + +# Update `syrupy` "snapshots", run tests to verify they pass with (possibly-updated) snapshot values. Just a wrapper for: +# ```bash +# pytest --snapshot-update -m 'not slow' +# pytest -vvv -m 'not slow' . +# ``` +# Can be used in conjunction with `git rebase -x`. +scripts/update-snapshots.sh ``` ### Linting diff --git a/README.md b/README.md index fd4cc82..2ff85f1 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Run GitHub Actions on ephemeral EC2 instances. **TOC** - [Quick Start](#quick-start) +- [Demos](#demos) - [Inputs](#inputs) - [Required](#required) - [`secrets.GH_SA_TOKEN`](#gh-sa-token) @@ -12,8 +13,9 @@ Run GitHub Actions on ephemeral EC2 instances. - [Outputs](#outputs) - [Technical Details](#technical) - [Runner Lifecycle](#lifecycle) - - [Multi-Job Workflows](#multi-job) - - [How Termination Works](#termination) + - [Parallel Jobs (Multiple Instances)](#parallel) + - [Multi-Job Workflows (Sequential)](#multi-job) + - [Termination logic](#termination) - [CloudWatch Logs Integration](#cloudwatch) - [Debugging and Troubleshooting](#debugging) - [SSH Access](#ssh) @@ -44,13 +46,38 @@ jobs: # - `secrets.GH_SA_TOKEN` (GitHub token with repo admin access) # - `vars.EC2_LAUNCH_ROLE` (role with GitHub OIDC access to this repo) secrets: inherit + with: + ec2_instance_type: g4dn.xlarge + ec2_image_id: ami-00096836009b16a22 # Deep Learning OSS Nvidia Driver AMI GPU PyTorch gpu-test: needs: ec2 - runs-on: ${{ needs.ec2.outputs.instance }} + runs-on: ${{ needs.ec2.outputs.id }} steps: - run: nvidia-smi # GPU node! ``` +## Demos + +Example workflows demonstrating ec2-gha capabilities are in [`.github/workflows/`](.github/workflows/): + +[![](img/demos%2325%201.png)][demos#25] + +### GPU Workflows +- [`demo-gpu-minimal.yml`](.github/workflows/demo-gpu-minimal.yml) - Minimal GPU test with `nvidia-smi` +- [`demo-gpu-job-seq.yml`](.github/workflows/demo-gpu-job-seq.yml) - Sequential ML workflow (prepare→train→evaluate) using pre-installed PyTorch from DLAMI +- Real-world example: [Mamba installation testing](https://github.com/Open-Athena/mamba/blob/gha/.github/workflows/install.yaml) - Shows `instance_name` customization for version-specific testing + +### Architecture & Parallelization +- [`demo-archs.yml`](.github/workflows/demo-archs.yml) - Cross-architecture testing (x86 and ARM) +- [`demo-multi-instance.yml`](.github/workflows/demo-multi-instance.yml) - Parallel jobs on multiple instances +- [`demo-multi-job.yml`](.github/workflows/demo-multi-job.yml) - Different job types on separate instances +- [`demo-matrix-wide.yml`](.github/workflows/demo-matrix-wide.yml) - Wide matrix strategy across many instances + +### Test Suite +- [`demos.yml`](.github/workflows/demos.yml) - Runs all demos for regression testing + +See [`.github/workflows/README.md`](.github/workflows/README.md) for detailed descriptions of each demo. + ## Inputs ### Required @@ -80,16 +107,19 @@ The `EC2_LAUNCH_ROLE` is passed to [aws-actions/configure-aws-credentials]; if y Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): - `action_ref` - ec2-gha Git ref to checkout (branch/tag/SHA); auto-detected if not specified +- `aws_region` - AWS region for EC2 instances (falls back to `vars.AWS_REGION`, default: `us-east-1`) - `cloudwatch_logs_group` - CloudWatch Logs group name for streaming logs (falls back to `vars.CLOUDWATCH_LOGS_GROUP`) - `ec2_home_dir` - Home directory (default: `/home/ubuntu`) -- `ec2_image_id` - AMI ID (default: Deep Learning AMI) +- `ec2_image_id` - AMI ID (default: Ubuntu 24.04 LTS) - `ec2_instance_profile` - IAM instance profile name for EC2 instances - Useful for on-instance debugging [via SSH][SSH access] - Required for [CloudWatch logging][cw] - Falls back to `vars.EC2_INSTANCE_PROFILE` - See [Appendix: IAM Role Setup](#iam-setup-appendix) for more details and sample setup code -- `ec2_instance_type` - Instance type (default: `g4dn.xlarge`) +- `ec2_instance_type` - Instance type (default: `t3.medium`) - `ec2_key_name` - EC2 key pair name (for [SSH access]) +- `instance_count` - Number of instances to create (default: 1, for parallel jobs) +- `instance_name` - Name tag template for EC2 instances. Uses Python string.Template format with variables: `$repo`, `$name` (workflow filename stem), `$workflow` (full workflow name), `$ref`, `$run_number`, `$idx` (0-based instance index for multi-instance launches). Default: `$repo/$name#$run_number` - `ec2_root_device_size` - Root device size in GB (default: 0 = use AMI default) - `ec2_security_group_id` - Security group ID (required for [SSH access], should expose inbound port 22) - `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing) @@ -100,9 +130,11 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): ## Outputs -| Name | Description | -|------|---------------------------------------------| -| id | Value to pass to subsequent jobs' `runs-on` | +| Name | Description | +|------|--------------------------------------------------------------------------| +| id | Single runner label for `runs-on` (when `instance_count=1`) | +| instances | JSON array of runner labels (for use with matrix strategy) | +| mapping | JSON object mapping instance IDs to labels (for debugging) | ## Technical Details @@ -115,7 +147,34 @@ This workflow creates EC2 instances with GitHub Actions runners that: - Use [GitHub's native runner hooks][hooks] for job tracking - Optionally support [SSH access] and [CloudWatch logging][cw] (for debugging) -### Multi-Job Workflows +### Parallel Jobs (Multiple Instances) + +Create multiple EC2 instances for parallel execution using `instance_count`: + +```yaml +jobs: + ec2: + uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@main + secrets: inherit + with: + instance_count: "3" # Create 3 instances + + parallel-jobs: + needs: ec2 + strategy: + matrix: + runner: ${{ fromJson(needs.ec2.outputs.instances) }} + runs-on: ${{ matrix.runner }} + steps: + - run: echo "Running on ${{ matrix.runner }}" +``` + +Each instance gets a unique runner label and can execute jobs independently. This is useful for: +- Matrix builds that need isolated environments +- Parallel testing across different configurations +- Distributed workloads + +### Multi-Job Workflows (Sequential) The runner supports multiple sequential jobs on the same instance, e.g.: @@ -535,9 +594,87 @@ gh variable set EC2_INSTANCE_PROFILE --body "GitHubRunnerEC2Profile" ## Acknowledgements -This repo borrows from or reuses: -- [omsf/start-aws-gha-runner] (upstream; this fork adds self-termination and various features) -- [related-sciences/gce-github-runner] (self-terminating GCE runner, using [job hooks][hooks]) +- This repo forked [omsf/start-aws-gha-runner]; it adds self-termination (bypassing [omsf/stop-aws-gha-runner]) and various features. +- [machulav/ec2-github-runner] is similar, [requires][egr ex] separate "start" and "stop" jobs +- [related-sciences/gce-github-runner] is a self-terminating GCE runner, using [job hooks][hooks]) + +Here's a diff porting [ec2-github-runner][machulav/ec2-github-runner]'s README [example][egr ex] to ec2-gha: +```diff + name: do-the-job + on: pull_request + jobs: +- start-runner: ++ ec2: + name: Start self-hosted EC2 runner +- runs-on: ubuntu-latest +- outputs: +- label: ${{ steps.start-ec2-runner.outputs.label }} +- ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} +- steps: +- - name: Configure AWS credentials +- uses: aws-actions/configure-aws-credentials@v4 ++ uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@v2 + with: +- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +- aws-region: ${{ secrets.AWS_REGION }} +- - name: Start EC2 runner +- id: start-ec2-runner +- uses: machulav/ec2-github-runner@v2 +- with: +- mode: start +- github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} +- ec2-image-id: ami-123 +- ec2-instance-type: t3.nano +- subnet-id: subnet-123 +- security-group-id: sg-123 +- iam-role-name: my-role-name # optional, requires additional permissions +- aws-resource-tags: > # optional, requires additional permissions +- [ +- {"Key": "Name", "Value": "ec2-github-runner"}, +- {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} +- ] +- block-device-mappings: > # optional, to customize EBS volumes +- [ +- {"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 100, "VolumeType": "gp3"}} +- ] ++ ec2_image_id: ami-123 ++ ec2_instance_type: t3.nano ++ ec2_root_device_size: 100 ++ ec2_subnet_id: subnet-123 ++ ec2_security_group_id: sg-123 ++ ec2_launch_role: my-role-name ++ secrets: ++ GH_SA_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + do-the-job: + name: Do the job on the runner + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Hello World + run: echo 'Hello World!' +- stop-runner: +- name: Stop self-hosted EC2 runner +- needs: +- - start-runner # required to get output from the start-runner job +- - do-the-job # required to wait when the main job is done +- runs-on: ubuntu-latest +- if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs +- steps: +- - name: Configure AWS credentials +- uses: aws-actions/configure-aws-credentials@v4 +- with: +- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +- aws-region: ${{ secrets.AWS_REGION }} +- - name: Stop EC2 runner +- uses: machulav/ec2-github-runner@v2 +- with: +- mode: stop +- github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} +- label: ${{ needs.start-runner.outputs.label }} +- ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} +``` [`runner.yml`]: .github/workflows/runner.yml [demo-job-seq]: .github/workflows/demo-job-seq.yml @@ -546,8 +683,12 @@ This repo borrows from or reuses: [aws-actions/configure-aws-credentials]: https://github.com/aws-actions/configure-aws-credentials [hooks]: https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/run-scripts [omsf/start-aws-gha-runner]: https://github.com/omsf/start-aws-gha-runner +[omsf/stop-aws-gha-runner]: https://github.com/omsf/stop-aws-gha-runner +[machulav/ec2-github-runner]: https://github.com/machulav/ec2-github-runner +[egr ex]: https://github.com/machulav/ec2-github-runner?tab=readme-ov-file#example [related-sciences/gce-github-runner]: https://github.com/related-sciences/gce-github-runner [reusable workflow]: https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows#calling-a-reusable-workflow [file an issue]: https://github.com/Open-Athena/ec2-gha/issues/new/choose [SSH access]: #ssh [cw]: #cloudwatch +[demos#25]: https://github.com/Open-Athena/ec2-gha/actions/runs/17004697889 diff --git a/action.yml b/action.yml index a2aa38a..d81dd43 100644 --- a/action.yml +++ b/action.yml @@ -5,75 +5,70 @@ runs: image: "Dockerfile" inputs: aws_region: - description: "The AWS region name to use for your runner. Defaults to AWS_REGION." + description: "AWS region for EC2 instances (falls back to vars.AWS_REGION, then us-east-1)" required: false aws_subnet_id: - description: "The AWS subnet ID to use for your runner. Will use the account default subnet if not specified." + description: "AWS subnet ID (will use the account default subnet if not specified)" required: false aws_tags: - description: "The AWS tags to use for your runner, formatted as a JSON list. See `README` for more details." + description: "AWS tags to apply to EC2 instances (JSON array format)" required: false cloudwatch_logs_group: - description: "CloudWatch Logs group name for streaming runner logs. Leave empty to disable CloudWatch Logs." + description: "CloudWatch Logs group name for streaming runner logs (leave empty to disable)" required: false ec2_home_dir: - description: "The EC2 AMI home directory to use for your runner. Will not start if not specified. For example: `/home/ec2-user`" - required: true + description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR, then auto-detection)" + required: false ec2_image_id: - description: "The machine AMI to use for your runner. This AMI can be a default but should have docker installed in the AMI. Will not start if not specified." - required: true + description: "AWS AMI ID to use (required - must be provided via input or vars.EC2_IMAGE_ID)" + required: false ec2_instance_profile: - description: "Instance profile name to attach to launched EC2 instances (e.g. for CloudWatch Logs)" + description: "Instance profile name to attach to launched EC2 instance (required for CloudWatch logging)" required: false ec2_instance_type: - description: "The type of instance to use for your runner. For example: t2.micro, t4g.nano, etc. Will not start if not specified." - required: true + description: "AWS instance type (falls back to vars.EC2_INSTANCE_TYPE, then t3.medium)" + required: false ec2_key_name: - description: "Name of the EC2 key pair to use for SSH access" + description: "Name of an EC2 key pair to use for SSH access (falls back to vars.EC2_KEY_NAME)" required: false ec2_root_device_size: - description: "The root device size in GB to use for your runner. Optional, defaults to the AMI default root disk size." + description: "Root device size in GB (0 = use AMI default)" required: false ec2_security_group_id: - description: "The AWS security group ID to use for your runner. Will use the account default security group if not specified." + description: "AWS security group ID (falls back to vars.EC2_SECURITY_GROUP_ID)" required: false ec2_userdata: - description: "User data script to run on instance startup. Use this to configure the instance before the runner starts." + description: "Additional userdata script to run on instance startup (before runner starts)" required: false extra_gh_labels: - description: "Any extra GitHub labels to tag your runners with. Passed as a comma-separated list with no spaces." + description: "Any extra GitHub labels to tag your runners with. Passed as a comma-separated list with no spaces" required: false instance_count: description: "The number of instances to create, defaults to 1" - required: true + required: false default: "1" instance_name: - description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number" + description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number, $idx (0-based instance index for multi-instance launches)" required: false default: "$repo/$name#$run_number" max_instance_lifetime: description: "Maximum instance lifetime in minutes before automatic shutdown (default 360 = 6 hours)" required: false - default: "360" repo: description: "The repo to run against. Will use the current repo if not specified." required: false runner_grace_period: - description: "Grace period in seconds before terminating instance after last job completes (default 60)" + description: "Grace period in seconds before terminating instance after last job completes (falls back to vars.RUNNER_GRACE_PERIOD, then 60)" required: false - default: "60" runner_initial_grace_period: - description: "Grace period in seconds before terminating instance if no jobs start (default 180)" + description: "Grace period in seconds before terminating instance if no jobs start (falls back to vars.RUNNER_INITIAL_GRACE_PERIOD, then 180)" required: false - default: "180" runner_poll_interval: - description: "How often (in seconds) to check termination conditions (default 10)" + description: "How often (in seconds) to check termination conditions (falls back to vars.RUNNER_POLL_INTERVAL, then 10)" required: false - default: "10" runner_registration_timeout: - description: "Maximum seconds to wait for runner to register with GitHub (default 300 = 5 minutes)" + description: "Maximum seconds to wait for runner to register with GitHub (falls back to vars.RUNNER_REGISTRATION_TIMEOUT, then 360 = 6 minutes)" required: false - default: "300" ssh_pubkey: description: "SSH public key to add to authorized_keys for debugging access" required: false diff --git a/img/demos#25 1.png b/img/demos#25 1.png new file mode 100644 index 0000000000000000000000000000000000000000..e648e63ac6f8ff81382afb42836107130ce3c123 GIT binary patch literal 186902 zcmeFZcT`i|);@|N{RLD61Vj*|SEW^*Hhb^Y*P3h2XFhYi*V1@KN=!?PgM&k=tfZiW zgM*)ogM)ici2v)3EOZG6=h}0?lP6lrPo6Mpxx3l{oNaM%*ke6oq*PyP-+=}lyw;W@ zr6j%Gcq3S`SWfpj-#av8kr8P(JRgt)_7Tdf&>X%q!uBrdo7?;1LNP*46W>(FC^MC; z!TdxM5MS73w9HC|-&XKjuzCrIBBpowLv_$+vCit%`p#i_0+joyuoT2US4&IhXyQb- z?VBBDJZ60TqL)rg9Ly~o**OqJ>ce}njL4kuW`aZnDtUX_E_I18-J9;`ifDS%`WKCg zH)6r&4LOfR$q!{Qw%b{EI__+Z>6(}w$xSxsEV4O0j-MUww5u+1)wId?$ZNk%_WJ1M zZORWfzdeh@(N(+DUF1Qi)kay7gb2qK^r#YKKMofn`i^4^di#>7l_vIX1Nhu8OkZ>V z(`1L_zR`Y?Ywj=sy@bB&ym5yh?g~Ujus(ZWI$XV3Kf3j^3!cNQ-~i(8Btb67I&B*d z&P^}7NS{F|b4?m)gGcuM0TZ^%4Q!2+?bOwAII-7+IM>1eI5)6Y*RUU2><2qCac^-5 zu)nvlABCLj|9*;}n}hrBYuxi+FUsjYQC7zO>RP+o+5$ZsTtSjtqu5l}CINazAR~1( zacfr>-WN8mR<^v}E^fb);7EFlV=rB7K`$7+U7Ud);@(nBf4?D)z5aEZkBRZ`S3piu zOh)Qjj89zMZ5c&)`FZ)7q=^|B8719q?8J2xp8k^@`=1n(0|?|M&d2BF<;CkI$m{BE z&-X}7OpK3TfKNbx2m1z(hYt|+!kY)^!ThgG{+*A4t%tQczzqa&1v381_l1?KCrFBk z=~qGj`TJKrZM^~iwA~^J=ij& zAMp!3lKeZt|KrjBE%LvT8vR#N5i!yKPWoSu{%2Bs4_o&qt}fU@LDK&a2iopBM`1Gfr$y!g8 z30Y)U=`r+}aQ4^4?26qM)z7!hw^zp38=9O9?{wq8*1L5}F*fQt)2PIaf-ow=n;pAZ z{P9x*z_qvBOF>y|8wrB;c7YO8q`QND8o{dqHPAP&ajp|GzWVK#EFm6~{3Mn7gWtdM zEA4YULiv{u{*Vv$rBJh1jCwuFfScED+?2)n-PbF|JBZ)b=I;lxJ=ntDWF+9<`h70g z?6DOh{oe}xwx9pM=pmtc(x*?UlJmO5n?IZkwQ{BWaR5In(& z@4K(GyYH=P`<$aMr}r;*gGJWdC&X^+7DcvliisI7h`PqX<4yNJ`s=#55|qx$h`Z3;}Fh2{v(O!60**b38yfC<5nKTY{i9mb8+UqGnUSQb*sUNle#qyo4*HvPHw2?@y+yAH5I;Ye=P^$d?;#N5h2jSThvCR-Y1VONu=Wk46CeRSD-&|%W3 z5BhDo^3e;qpUZ^5mV%iyePtsd9mRhURc|oI-tHUz&@kM*NQKR0$_S{MZ(;E`%q;Kl!ruInBg_ee$!?iN5pXoa0ryH#EScR!>H#ZL!KB)(sSeuTYNuNSLpH*a`;Az1=^ZKmEY zUBCV*kt+!g`S2kO8ckyxZ3@jD#hLm3g+s20EVD&vr4lt0uqt+60<0$Oy`rwHG-;hI zu~O01GD%E{YT+?yjJ1TQC)TxOuJgiqZ_)_^IKRjo_n0pSM0@5=h&4Uqqml3}plgc= z4^MJ(ZBt}oW=0?VT-3>ty#}yr;TcnVG%%a@2dVn0wF1?dsRiA-Jnw&7OU$pj{bCp~H5al=dS5`JeqlQKvBRu_4*(05$es7zl zb|4b35ZMGkNFzeDeAP@BMUj<+Wwh1<;8Jr)7O5;?Vx;(%<>Vp&exQaKE-5n$I9CS< z6bUT?z9wh)E~}>(JgXvcu}j3y5i$kzEp02{Do>*=(S7=bdU^>P$W{b?1}LM7`B_v? ziZ*k9BV9oh(UG~Ewk0OuWzbZ!M;JXtSv%2N0FH`aQ{dT)IC|t-EmviY93QEPtq7c3 z%5UNi8=~wMNvPsYl@JPzk$K&QHH3JboMq>|i}$1kcop+a^3YeK&m*Wdff~@Kqul8O z=^}k2`OnTB?STi;#~!vNy=kKYbD(iVy|F41{3e{z#Ve@GWCXY=M55e^r=}ro7s5wC zeqRWxryhQXV*yNe*ZDMq7r~3;DakEGMk+66tsAg^%s*vyUC*TGa;Hvs^e{gXBkzi+ z%-jO5q-+uIPd50=yv!h#2TUZ`83V);QrKbYO`&OWMZv|ke#uI8R6LvFr0x}>sC9sG zt#@n!xACZ*7GPpbh@y?lurk9OvW>0%mh0s3VSz%gPMkv##~&03H%CHjU-YFP3t6Ak ztM%Fn)2W2;UZ95*NtT|)H^-y$tZ98-lA1rYEc;z;6?z(DnL2xKWQLa#g3N&SOXXP`V$Wh^x_O#Ue+Bnxn z?Ix>f6%qT#0T|N5w>|>!r}Ruxuhyj-8%B71q+=wBeL1&yg%=8~+|{DfDTn+-m|; z8>xCNn=|d0l6}$nvLm3;f++r0q3zFXl1;xn^J__5mKH)G>OqaJNY*c50V(UoRtJnn z1i#awuoB9h``1d8B5`F@!PUtnQ`l6WT(sL&OM7g9ZxNW7fy_@nJ4a-Lomr*u**@l_ z%Fp(v_w+2Zh(9dUUStJIkX)v0Ypg5mCRL~(f(jpsx70sOSseI~w`hb((dg~%t@VI{ zTO&}dU}d#b0q_Q-|Hf#ohZ&LIfl_L%ts-<`2Y;K>pwuQEc`#(qWK4gk9K^)LR8UDH zTIv^_CGtfL8gYV#Fra+Z{vb_ckBAjQ;kkhMUQC)Mu6DCNNRpYe~R+`rr7e%K4+mmK3veEmaa)!#zZ^D774UNaAk54mp{nLgkkLn+r(`C)AGmTUp z2^kdz*P>w;jn;|)deTAdKWGZ`V>b!Y)!4(hngsJQ&eiV8Jgsc~Nd-W3#P_F4>5?&s zKP)q>NI0NxoqU-pWaE{BAvn(42hq8#vOiWP;UQmCXT+LJU zwBCe!k@{$%@@o%y6x9s9(J(`|p~Y>M2sD&|Rczcj=9ysBwAY4ntHRiBr5=X{XZp)D zAtwN0kp5Sb)Y8Qj*T!X@nAAHmQ>r7e+qBhzKHTmZU?Q2~4i_aQ4|daz?Mp06t30Mk zaV~x>a<}Nzyng<0V=HU(PYr!Y5~|Od;IT#fF6~i3T5pu`3U}W@8N54dcTK1uRcn2s zveCT5Q>F@H-sS^;dW>Ev5y5>1RUfT+N5;b1SDFah^o4v_j1yoT{khny;lZ&U4`<*i z@qrmM&W?6o?o17)ih0k!+Tb@|!W=0NepGK6oT}0t3YsUfr-t)YL~5_Bnjyb(`s3n3Ee#%?-8rnp+zjW{`4#v4ixu^2Hm1E!Ztp=$@I=mf&GIVfhh`zFZ7M&nmW;pezRY*`3*Vt-dCv){siKzCH}*@{;Y*HqwaQa+`IiJ!_BRfB-GZ+ zA0o~Pjs8P*375RCz1BKDC!Rn1lQk7=c9W4Xk)&AY{r~0_EPqV@YWLZ1bwI`zP^5w) zRH>#%iQ}f+1-_xe$#rOaJX*^tZ$o{OhsX~Mu@sAU+FhXS|FHWb3;4Gg-cFM8^!GV- z65io4P~-fjPx;8JkDOj3p`hTQvdA@j;_Db9g%nxaSSG~+qYr7ZZsqg!fuuLJ)O;9Q#whA{v&e0lc=bqP8jpcS3$khIucO&-@od?*7NO62crOa^96ZR}4rd`u%pzO= ztz6Pj4K7~2D}g@-@Nv)3%OYG&vGO&yZ46x(P8r`-+V&?YY!awKBs$z=L7%2v`>v62 zdq00;EEw&Fp08~O`ywYl3_4o;wtY8y5@fLhI~DXisAx@xD7c^SAq$JLo*u}qbXxqo zFR9(Wc8a0y4cuOmx8Xh%orPXXi`o|gX3>T?igC#|J}-FOpy(KN2si!LDrat zCbTh@tzv7h6fpg~PusudQ{dI{f-hi8^JfT z@Z;QdGB$g|``$Alnf?Ugh*1+pFzT^i*CKm^pBJ89Q!pXb`PKNV6WI-wL{1hZ} zJM7O*eYU3TZN9m}H}_5%$Pz#HC2sB&YKx zZerX~xW8-k{2hv(T}C|>YJV}9gK?zwWfh2knRs7z?~ zu!Eg$cQ6wX5pA`&We?|!@O2I(5W@KiFYd_%n&BU`Dl8AB)BPCE&sSAlWL2N>B&8E} z57EkO0mUi_dS@r$5|G*44K>+A=x`X979f+n7A8lm!8bqg{QS8y1^Ht0)zEU3p^TcH z1TKp5g(xKR@>qA$b8sc`G?-Fx8)15jmpsbRt1#hl z$b8NPGIi;+>HYNnW>=%N&ld3#fHlF63vI{YIxSPp?bV8o2#<=(E>cgf6ZeX#Fl}01 z#)KAZW>K4^6lF%yOB5O>AxMP2tif(Eqqii0$uF&;OZ(q51WpmJ;*yq@Y^EBVAbvCB zPGi+@Kkzm<854^CuCnFe>0!k4gYvnhA-tf;{zh@nb>*svlWFk}X+rT0PKchk?ueO9 zRmH*^KAx8Fz_bP4!|dVxlP6bN`|}_Q?-LB9aP+;O`(X3tT&JepP!_Yy8NV_36cz2U zwiJfY&VPo?O66@nvNEo-`!M4*7ckoDrQbTw$o#mmeI^U_-Wp}K zM)ezpK%2MG@5>%O8Ca)tn@;XOKw;DtLE61&II)J6~&ikg$8$kv$3%h9SKVG{vFzSm*MA)3{S=t^} znm2>FA-Z3%U0M%`kEw>P)R*d209Ts6y6zZTl!46=erpNawih!~TPK~#+)10rrr!M! z*9I7^i1R&uLeEvLJWQ=?QY$^hm8Rknb+E^f#1MV$SHD4fuXQ^?Utl}hE>+Yu%Zn0@ zAv|mFt|65_KR=wqj9HDBJdClIE7r1+1N;D>b~#ctgzr1v;lD|_=3Gn(OYW8Onti~~ zahau-A~Y3HH6*~msd00yJJ~YPaaV^EupXn(-CSHRj-J+Y4{q7ztxY7%wyE z?LI$&*(Q_M!pQGSvLnm`3imL6HD*>xqJRmtD0(SQIuYlXm8<~P)w>$N{A;XRAJ>6i z80==N$h>0a4lgiU;sKgx&q;GcYdyag@1Gji2R|+^Et%%51k_gEjYXrvKYu}Ji^*B|v z*|g}k>9(Zvz2~%Z&d08Z9>^Ez*rEPtDs`fOtvAWehp%G0nQYO0lmdVS1q#K>PnCz4 z&srakSvMcW1y0l}#WKZJ+edQ?GbHhK{AgSCq$qYc3_+L&<YDU#`g@S$Qb>Znt?W55LFF%%Yk~awj;9) zITx7e*kl-S(neM-05p+fK(2;+lR=+4cF3<3cE|ejkP?~RNE*R@+&R1gj}_k0jwgN` zW*@8DcDMXSTl8UPeR1gBz@1(;wB_XqwG@}FF|WU9PaUJ4hH~3Ac9->>UNCVIX(BuT zZ#uBoGANaCFt918ciMC9Rk|n6_#ZNE_B|jjyiv3 zqG|7ZJe@sVs`T6PT*oLh@1Sj$R$b9(YRi~Q+ed}fs))Lo#^wGj7E`r(^eBwC2___) zADocSI-Hnnds5ZegL^p3%-4RB=682(!SZR;eQPLshZoTyy~t^pwCkI}7gE)!QFM_q ztvG{6ZEmCUNVlc5%y|v8HL8e(DTh&Q8YL||dxAqMWfK+Xb-vD+@0Nj^c7y;HdR(<2 ztkvci60`NzF7KmH6=W!b$F|={Z8i_8+pI!Kx9Yq1J!}a!6y|3L+w_`cIh?(GMdt=j zoNmv@9~TF&Jv6ikXfT^Om7i*{jTi!72x39uK`bJ)-j~3`)iTp51e`Y8dj=aOpN&mq zecb4@LgvTA#>TK)1zZt6D(SwQW;ZTPFz;}+JP;&x!9BYYlm5iP`{R_M6IQG}H(7UA z%)Nj}!81wp@=(TOs3E8R(II7~Gkd$`3=Aabb#W}U*>1auw}y1>OmxZmi4x5FxH9Wk zXwu-g(?!70GqpThGOKFlEbuPtbSP-)Cp4R%yf-;9*}`92BwOkF@WFlm)dz5w=PW+E zA4HuI-tDJ*v2{bzrhym7_NV*-mlwDNahN_}qL1uF^Ga47Y*d1bz7Ft1J)|&;W^}P< zgt>O9ie=e*HaSy5H;ho(_4N5VuaFct6W-p6p?7MDyf55WVK6bXyUaDatF)2~p{ePBfh2Zpc>4>k*yvXZAZhnT2iuX2Y_U1B!JT!kw{jC9CFz z1#6nZ4HXBx9d1JX8BlyOYJuU^XnE;g%K;3K?B5ktr}RBx>j-o{!kkMw>tG1FxW>Ny za>irseCPPbkN`?OvnXiov8o2mL9Z4Te9or}4S)BJUoP5dDL$7$On+y?b4-*OUBi|g zmy&YV@2|8sx&t=fohPej2Rn}^Bmmd7W<&H_57j%f&rc4h27Gt;Xz!M8bpH6-W@nfg z6v&s2lhEWi7r__X8_(K%7~fGBv9D^~rwJduk<*4;-K!fqH#|rsZzfuVPU)xp^qj*b zKhcA42yIS}aL`7ce7k`ZZ_p$)5?^nt^6hvQb+4w2oAjaM2xAd1JP*2h$9}5D8fw#e zYz0???5XJK)j#ekqDKa-Q}Jcpc0e_WcKOStezHISS(Rv=(vj7`58Rd=dJ~lviLBm< zvlm8TR~Z7IFpN3I+P={m+o*BTCv6TlRkpjL=wxQ4L$@-l7g;~p{+LfMu%h zl& z$qRCbpKmL^PriNUx8le;gI*4K}v z{QCAq&9-@U7U_U(+Jh2@W@2uPg@zDAOTPj9?-?rMWQki%ttB2qs1m7gvGp(C_5DUi zA4VSUF5^pPWK@z*IuKGl4bA!BEPRXIi9?XoRv0HSRisY$S)EbFqg>FbH@v|Zm_W4n zEwyqL828f}8Xl#IzH|PmrSHZE0aN_B41vn%2?RJWO?({uD7l)g%VmL>f`%-))^6W1 z`$>teC<}a}R?e{JjolA%h%>51A3u$J72nhGy&WA38yix8Eq_X_OU246@9fm_vbJ&k ztYr%>J3JfOzVkIddU9VpFtKTmpTSDw*lm>!_%@L@B04(RZ>vSuHw$6jAwZ+AF~XNF zI_ry+M8(8Ofd)KF7+(AK)ABoMH6;l-YGDYwaGEVqssTIah&a7$!a)Pc>hsy9ll!ZI*xO;<)hppfL# zny^#vID#}OjWxe2p*LzY`xk1)a?Tgu)im>+;>IrE70s1yXARxpi}Tag!*N!PjORl; z?T52X*NVS{#V^fiUYc%wJO@wJ1LE&+nkl_af1pxpB7`FxH0$p(Vk6bQ(+4v~%}gM0 z#y;bbpktrij|ISsrfd}>sA5fl>h0MZb3dMtOF83e!IG=^gL6!mI+w5cOF^t+yO*da z?h<<*udP?P;5Z&`N$u>xHDQ;jr!C?Y!0ez{U-F45*Nom(6YDLEBdV(7tPdZzVWpN_ z`fS)>WQkzlI(&zLLT%luc4F4&)J38S#+4RXyCPYX79 zV!9xHblt)SgoP*TlLPV5Gj}QP%l5{0_E z{!JHEooewBYP9)On>t#!gr^6a9*C9kuiLvT$jYA?=MvVm`&)2?@xwGRP~agj?)kUZ zqj@M4;O)>ZUx&@geuyFzOq|d+z4F>Zgb+d*#1BZ~nke^RK0FEi+>B~zL8>3WsO7e7 zpD>>*H&*cix41+kRqSHcq+?bFyX$Lv{Y~m8dZ2nUzeOH_y>l6cvB|73rkZj5?hc(goUi)x^H^nS-)R)w9h z*puXbfxvjRX#JMTW*SGk@1Ngn0rxXb4%b&^4N)D|turs7^g2}*0j3a(^8`Z|kC8mZ z#`KQESwGE=%L_l`?3Q`Qp&{z+uA6*_rpAaf`(?ph*XMl_o4`dPGdz>+G1vzL76`$* z&?0tA%{OgDaT0ZfNatq+mKs6n$F|c$d2)GL$%nJ*?>oddqmE9vvKbTbjRbU0(#b!4 zX64*S;CoB%#flC&obw3LqJ20LSz>76+hv|@NYV9;t<$krcM>q5C*m69Y<(f);jTW-A>=@`}n@ z;(!v1&<~!tT7#d%7~1AYmgLC{SM}%wn>Ypgy~ySI#zhlsrueEze|DD9TU9 z{u7yMNfuq!d|zYps@Tym^+wpcs*e7(BAN+AqhY$-e48RJ_w)xdUk04nlfn9?by0JJ z*+`@Zid0m*EWqNF^e)!LRZPWWdU6kO;>nS!WZNGWt}BGvbBX_)W`UV$R!w8nlmDD0 zlzvYtNxTTr6+!?n*A#!$5$C1Y-klq!~zsBn_)SKZnA&CZriCPAY?>(9x zCtY$NF7D7g$!CaB9Y}s3HMBp_a|)W253AB=CCU-q+#{eo4FBv-~Wf4U%24FM=c5PMm1bm@>;1oTWvubnRrqwzDNCr zur*AA{IlQsQS?Yc0%)>T+!U@{L?`8{tg@4NSm{wDBdAQ4mb-Mv{+_SBbV3!S=FIM2 zprM0L!671q`ZZQ~+2&h9M0m4#i?Wz=u=?j>=h}BduY}!XgM3crE*n=~;2PUWveQ#I z4y*}-GdR@M2qAfL1<$SJFfSeFF(|JK>b;oE{WLsEb|p&Cmz(9kOf_F~{5bBkc?0{thdJsAhI4?^*D8TUWyP42)&!=DBN{1aC>1rPL``IrnENsl%{Y>|; zkL23!7)`I#g;Wl$Vfi`A^_XdO8TMKoE43WXCY3^!m(uF4a~&xA+0+r)CbYnyZf89e=gd(lv;QZ^$ zSw%q6KFl6^;KKT)JWuSIWtmW-@s-HxCl2nM)W_D}B5Um3k}#wtF33jraec#H0){qM z1|7u4J;>@^g^Oz}_L&_Al9&G5&W%M*X)kB&y|>J%Im{lKZXUVnLI1)m+{nRNNFD-? zW1`E_i`pYE^LQCR%|ZM3hK7W@l1cblwE7a&F|)(S95^}`VSsS*dWSU`H)hz(L3zlI z{dELP*wE6Y3?TYGgka9b7q#|$D|#X>tKGNP8*V+0G136zsF3QO!256qHrZNcNE)56 zh~T&QCL3YgzA2H`MLEpDv*>$?Z1?@Xp3?}s7r@Kg-R8X4UA&}+L!)YzEZqh97drXq zpYd(gPeLmT{tk6 z8m77anc|+|njQn1dziJl;eLl7{Tx`D0f^OhqCeH$Ei z5YTuV&y#Os0ZQc>tM2O^ohMnRo!f%~=Q@N!((zG?W0#|H;KQ`8eGHW0?rz3@P|M}l zBda&WZ*1F;ssln-f)WpId7srlotHm=I}gWnFKb;Khm{50>m_$L>JIcTN5T-D7rvqX zq8LBXkBgQ4-@ay{IND|RH&RF8u_7-qg~`q4*X;6@v(;%u9Boz_L?KwSXc8wOD+C|t zo`1Fj<7=GyrS~nTyMn!P<(MH*{!1WHy0DYuTUx;8M8GlHaHIZYy?rDQi$OK9K0g|i z*q%m{P;tjRC_;XH@;+>%?ckFlQKqcX9L<590(LKL2-qkZ2Baos`7S)d_$kYHMY51e zI#4@BQ1QAQE~?9vTjalOok?9CE2$GqU9Y;h7RjJ)vie99;W2n^@3GAOladuY>b`t( znWb{^k}A^vS6&PO0{!;co#!WI5>>dn<07$6JR^mw$VeuyLi^7&RXxZ*c1hzlMG018r6+eBecxXwd#!k)~Px@3r1ZcEkzNuj-`Dh-MMG?Z+@*6VYyb$H^ z&t-^4CMQftQufK0R$D%6wQ0gmX?_E5T(GPxXaYR% zTH^hcLw+e9;W#mm73I%aQbmxw$qbQoVAWxAcHjxXZ)kS~0Jnh`sq3`+?!Q72w9uiS z%SQ)&J|YsKTbp?BWC@EM)m4ZE=WVlU)c2BHv>vVINB8eY>D1Z~t(UD^irtDdsdvbg z{JBIiaRJE=O447xaIP?EP{1{x7MvE%Qn+&oP918I&9apTfa9V_617KU_40E!yNQ%+ z_2z=w;f2|kI~#-RDQek4%XANY_V-2o(@dMd6%ti6j9w^@D!O;WISSbkd|AtG(JDY~6mj;wg_S;v@=88ebQk&Zcu!AzePw`)j0s zg6Aht)v+kmUSmY*44Pd8Mi;{%XZ%xH+4twtNYgilQhrios&CxY*=|d`ZRAb`oI8jX zOkMUotjx@l8I*m4B}Y{64({~yBm~+}Up5b$cMR=uV3G1d>!r1Ie5BJDSmqd`1?O{I z^YK|1hr^P28W?)SgOcVZjWjafen|U>x>;!)pLT7ze1-BVw^OjLx$HIs@X{i4_c)p6 zHhOi$`e1fjaHzmWTK!~8!sll<^=68uuV%ePyRV3B?V#j_wA&{f34^w}0N<*?jtoOr z)U*3OMj5jy;Z!_UxLTd5HKiHcu=L?(@4Z4|3ZLq<(&-Xe4;lerenUG*?y; zXPQg(fG^Bt@xm>Voqr;|`aVZq2AutbI-jPM6m+qji#dO*O*=cc@!1=Brc3vD*1E5g}p&M$ig z3^m8Pel1Q(thh2mgRuUk)**|hAMSk@+|E4Si;pgh$3%>A__jR(YLFH`hCSJz&XL2V zbt=!4xc*hp3B^JP!k$0_p=xxA4|8n2D zpyp+E+i6!Akn!jHpmP{L{eF9tV91OQZlvnPiE2GeiigXf{H8G0PVwc%TN7}+(xpCi`h#ID$+>rVp4XDIm}x@L?-5b zVzk4Zo%GKvLd={d%8BmkHSbTDUT^$4A42}_qQDT;_VBz!r!bsU+-0_H${=VjcFg9w z`S9nN!@VT7Y=50Sw)Wd}>2VE(hv4>ZfCMkE?Z)xZQ9b#{;acJ}G8t&v=xn?j<4-g) z+i(U8C8AIW+6x#~<88N0A3yBsRA0s=S;Eqi>vDy(5>VWbPmqKbp)q4E4cH0Ek68&< z7`@lrCU@NB*`bdp;r>Zkl~U(5+yT;yxGmq!+5zQTGtJgvLn+UK^S*4o6-)V8?8V^a zgo0xAwza`iO$QuxD6@M@lEnH(-FaEYn2%IaE0BRHzC(53_9*To7IEHd&Pi6wknEP4 z*r`D)2DK>;847l|pN1@k_0GY*kk9Z9b?()+o))jNk1)87Kxj_PQl|OV;-jTFvx9^` zONEs#lROJLg9*ixm7*>R4Y4sI)8x7pb+Z(qrOq8U>R03OiHQ6*>Mv!m#`&IdqRV(o zCWfyeGf3F$!Xuug0rNHNAjS1*kFe;B4X!cmjn?l)4-QnJbl&j87h~`Ema;2MXGE@M zoHKTkYzVhgp4;9x8LG%TRHWv!$S@s1f*ZIxv=Xhl!+WVM{Qe(P7_z zvNV9OOkP%}Ey7YBk(Fl6^1O6z@14nZ9yE0zOugxdOLf6cL`gIXg z-&$u+@2{TRvJ7I0P8UMlh_A$0f-LLo2jX^Hy;RjDgcFwsGMVY88>u##9u5CojlVs; z>}hC;nX^ed-eJok8wS_YNkhKsYY@qAPSqie6PCy6_*LozK3(5#-p2YzXZ_YmDIS`x zTU5;+9I!s@>8W~~{-|m-oDYX3;wl5^rtEy^D^}20sX1)siF(rU?n;DohNhU*4Pu4l z_oVk}9=~5PTFiZ_lh@&Tpak}+x>h{X_quQG5ha$4E)Lpo_=eR#hQnwzir1gDJjOZ6Je zA8!fB6*PM+r(9SCgakFaS3hv6f2CKh*9(#}Qangu+<3e1q}k;`$U*qwDtb+5vuyiY z_0=vmwOMBRqJkG+-ZrPvgKIS!39Acuj_4oFgXkR$qk)j@iRXG=#q1$9wayveN(Ob}msp9N#oP{6(G2sv((nC?`U_aTw{=V(uevTFNL4vq)HO=m1gEICyJXm$rT(i z4Grl`%~TJPU|9u)PU_F!mz}VH&28oeCpio2^rM{GQ9Mgum*sBZ6uOn~QqNv%S&$a}o;~#| z-bbEE_Oa;0NTJ`)?_ceir(`5Vk3XmV?dbh_!1(a`ja>Q9O3c6Q-M=4LU@4COZ>n{Z z^M9*_bNzpN>u)0F|IV#{7S;c*#DDEUSh@B8X92;!l=-xa9r2aw&#}3z_2bxXRTb>M7_X+M)%eXd=}Xvcbsas2 z>)Oq?Lq~D+u=Z$=f8Vwt-wKv|@57sus(pI64NDGYXT#C2c*-+X<5IRU)y_3EM^1Jn z>pR$ulAT`nXc8ZA(JGF@%`FA~0ux~xpZzMxnqC4@x!7v(x} zZZ=?$`ikw6QR4c^&8AsZxSyj*hM)5k05-+CgbhT(4d*?&XYq^b9Nb1dJY;k3JjmCL zu@>PJ{|4@QhOZasvm&N>#oP$aAYN@3qMwl8t^mB;YOrlWsS)Hd<*h5duaizHR$WFK zW|Hd>QxD3BisNR={8B7}-R|@lW)IliV?TF~4Ci%$z9NPmhFHsA)pSNPJdl}Zdy^fm z*0P1EhDuS0RxHo`_}8{+lL45rK>|)Z5=!Fw6x+XNo)3G>oOHUd{^MHjm&%eKWyr^; zgs&VjNY<-aUAUhUTZ4o&8xGp-ct`?9L(&&J{|h1_WC->@oWg40w~FXKTSKGX45fdH zy)r7vjhJv%N*G(YJS{QlX*GL!w6~_vzf>|=UXrCVr$@=;G0Rs@(#PF5+}(#hFphB? z*m->XyY?8ur6syYnRdp1udDy5*~xZL_H;=#c}LxN{O_{>s!8R$^-H)<;4Pb}#|HuE zuYZ?R%NeutAthwHyL>C=K>MgG7g3Jb-wzfK=8Z-N| zC7@^h;N(Yti~wf5*>U{^b?9h68>u*8{bx=yp5c9F zB1A`?-=Q{CT)>8J0S^~MUBLT7cd>5!5%374i@ibyxX11lV8GUw#KUD7d5?qA${M8} zk<9)8OVLaeDLQwR04H4f?I?TwhKl;1aW|%T*JH9ji~xTNa=)%cYqo=*lOeTM>veh9 zvYDs3KI?vJUKG(Gm^p6j8*o$uoxvHaEmTGCBP<;MG#BH-yFh^GzJ<_Yk z78m~`mkiy(dw&`9v-j`b#lWiWKExL4iVfc-Z}uN%6mwrHp7CEK1)`!d#d}FZ2$09= zz>jiK_oX<8Bo8ybc6ayk!Hasg%v){d(a8~Z<^$;xhFO7Y99Y75EF>$?WNo5~5831r z@A@<<=28ArqzIGho#dw3K&VFN=>hVW2L=Py@RE@69TOGUSd1?8iJ)@F633LByTfaN z113J}1ENbqiV@EAE7=d4=73EVS~YPSlO5_*JZ9{C?LXS?NhCS{;$><8X#~D%$XG|A zlT2L`zp1D7^xU#k9WZGq49BGLOG{ha4_dkfQRbl6HL>v7^pue4vx!m9;O{br-uP30 z-c(}5lBP9Sb+JNZkmv|MZ!qCydhLX;W$<-8XI(5hc-v4+V?h*(Ja)Nsj!#lg7jLbp za+z+7O{y?C=7y)*4WzfiNT_+NLbOmVUdansyQfuy{(V#wxCw)TM2`suL$Ja9N;WlC z;ASx1B9j6-?Etwusbv<_O4USU%_#*}t zW`OJVcuS`YL2;9lJ3#tOAQpJY@YwPhU2pT)d?eyHI*A%$%MN7K9?@1iL>K~o1S6%drc?hjhp{k$jHe?M&fIFn;dl7wM0}c zmShZuGbxh5zHYO@R%;fQVxbJ!V$y7zl(RR5a2njZi7CuZN41YdUqfrljb6uTJf*j71cK43OFhXKlVKWCt7P0NX*sHf!bI5KnRQ zWYU1T6M;Bu$yn#c<;9BeNF{!Q1OJ*CMJSe}_-b@9N#ma)7r96glo|4#|E~LKIiCh#eetI!sOjqCrBA3cTnZa4FbV$!SO zG_QI{M`zkK=y#ba?2$ARutE>kKE@s$u+YABRW;2*mmBuB26nNLLy!Lhs zYpR5OB{r^o$fX5~hVO(D^_zK*C~no3Ym}Qd4Wew$TlTm1UgHIQgd_V?v`jK8uc0XI z77mYhXQnDl8z%t+@32y?g>l425x}8WofCk%-Ki+zI{Pqyx5KF8_`nn{JQpqCv!)3= zRV1cv+;6-5>vV}?yF4@NQ9w&Tg?3pHscfc;Ks6|mn};FrM@Imf;g6D>Ik>t%bTlL? zdTz;1da9qDB7xWw+t)>_(NqKBqmhMJ!8LWPNr zc%#^!*CSp^U2AjdQhFOd5@h)-+Ba(P>tmzYiQBgiyzMvl!nQlvP(xCPM(oO3fhh*~ zBGG9N4iF37fVH}Rcj}dx2jRn;;8ZN9c8vFEk1e6l|9D4Wqwf0wqz;oe$U!ISoPY#2 zD@LVobMT=166>mZp>dmK7rJF&7q1ri?(i+pV>K+p2~77`MciNE?Tmk2y~C)&bc5sf zC=Odz0nZdz5^ubepU!LD0QKb8Jf@BHNcBu{VO^vy(SNqNE2^gzTummA{$^V5u|oC> zs1$MPJVO&*OIHe985;#T0%#i_va`NFM639pSi+mi-c z#D6}Mh4K0^FNF&IeNh_u?atFmhiYZj*t|C=oUtratL^j7z*_$)_g3J_VCGKHrEgzk zRf2#CJ5XzKjL)E$-b8 zapauW5DH70rEM*B`!0Ww@huex8OzK8-5m3X(&lP`wUMS_f*LQFv5>rHvc^GHS!)8( zAG%ML@uTdon9q`1(ewKc*W!)=6I=8s@qyaa)db-8(Q?A-oI{rLhrE^HSk>$qR?V5F zefd(q&=U!^3yhDXlj`I3So=V?`vIwv%F6tuSed#k#k9H>T=Xetzo8FZYe2E(n}M>t3F$}frw5G!=)7g@N!*PWT^V(8DH5l4a;C(()6CW2x=8M z3W||DBJn8>pJ{r_v!VekZ%mHu*12-$-WF0u9nXcCO|dl<=4|NUWKcEOTdC34h*bk; zxGOg{eq@ zMibD_E=`>F)y{;fZZMuPl>@5igM(L~3DxB>MDpZEh%qpP^vZSr&rw9dlBDt+ouo_y z>X#pepTj?@bC~ZITE;hsQ64qXT(0%RJ~aO5mh1d}E)Kptc_q*DpLM)bBAZ2!yHnyf znRW27#`?>_;kN5|5P_T241?uQpB=nfn3Uo-%t$ zi~fq_;xr#zOG9(UDc$Kk^NMNDf>R%X;|t%J5}%*mn^H^MZmV0$FGJ3-z1tj9A4BZ z)_OpLWpzxout=+yVL7OA$gP$k-k^)AphDy9WD=8lXP)s=M1okI-Q<_m$z$%RnsWx@ zMv!tNnQ91kgX4HCmMhnTJrQNpB_G)=Rp9;ew!m%^hGs&&DOb|M7y^_Ms|#RDwH;*U zz_@+NADN#uMh$wpzGcIn2DKV%*>~dUggY^`^?63X1sl7xG25U!QE8lsdSWecLnBT0 zG|Ifd15sBf8$9-W`!ilAY-#&jD^v>t4EQDe&YFLadl}oY9+~$<>RgoTN;keXlR26I zXRt)^aqmDOM@PNImmR`YZyq~WZ^8WKLd8ud9>a*$%IvG zm(NHi>$V3394TG08XrOZQzZfNMVipd-6|~e^q>XV;c_eo2=~CsaB&n?D3S=661vq)U}sfekaij`dJs_uOY9y6je9HEOVF%tm=IZtqv^u7LVNq*zu%)ha z2pZ?)Tlu5Nk znYXB<3ZQH6g)VqK;G0AJm^}=}Jw;c^P`40F%`1R=YAHllJiD zX7^XUc$+%vKL0%Xf=NkBCy1#R5~9Hv6UiS3U0~*f7r*&i-`AN!XIZqL*iT|aEqrD# zP>Yp4-xYG!*}~Lo{~c(Xp%9G4CvHXSUHAds%=xk}?AB~RQMnGx`2jX<9kRiMVu6ys4i2l8 z32y4?uX$S(xhp@8%isO6seUKm_NJxMH!QCxDaCf)h5E5L2ySqCCcXxLW z?(QtkB8xjL`!@fd+*ft)eXnYY+Fht+&Y77$eZJlOX~=Me1y$3E+?}C@D<(8(mG;Fe zqK#^tb-|MNjMcdyW9YOYW*wB?ap2%yL~teLx&6fJ7%Fen2~PJmG4j&y9Q@^dkee$b zb2!6dw>$Li4>S$xD(ga1s)#<2pS!1Pc^<|kyrnh?G)c=h&7dA?_fGHu^kpGd{wvW+>YX ziYAJ?+|aISvv6z=rK{}J^Ln(c1r7A;8c;r>w(FSBS6bkpcHW>#G5aQ=1Hf6x_4(=H z2-*`TQ_F7c_iM8=gX!LZZxa9Ht5vTS$1|3hwO-AhqBeo+)SzxY1$4!Y|3e?!V4e?7?zS7Mgv5)GRh7TS@b3;&->z!+}S=o6hF0xk{Buz5zj9<`sgBdx^pPeV-);?=WG{e_g%v-)rD+z-ry*$QOGH-!fr=D%^Q znI|od(ht%#yQJ!{E36tbjRVIrAJU$0k0)<|^`oq)0cPS!D%X!K)pryNzeQ2@Qt_=D z4|F@k7jSy@tOHx$((X@4B&O~;29Q5L)O7`Xj7H8v%i$Y^rRh@3Ql0f zbrcbBS#4Aw)4_4mNi0)sc$%lGhd$njnhm7qUY=-vDJpuz+)zOh&hflk=uYRl$LF2C z7iE8#*ox6(n_8}@6i>zCwQ_t5m)Z80CmyU5JbLX$oo>MEaZ6}QVIu2g+16>KPDheW z*AvWxyW>B5)_abOq4Sb-;cyv3(}rreRo3Rfonq?F#BC7g#ChT=WABqhv>s0AODE9z8CrWZ z2Egk|J9{v5YN&%;cP^$oxuDWd4Lr*g?uAeCH+rL(IrnT3t7dmC2X#W6x+ev|+TVr< z?nYW2Ep~a{`*d7w1dkn9vOG1Fcr_jY%Ankudcg64{?Ay`ySWv*Zl_t$)ew_FlXoAN z#B$z2pfVsrs`RYQ3v%pO>m*JJBhIIlYjJO?K@Tuy}*idNx%(G=k9>vPiaLo zf`T8GClQphXACDNpaAZ1qATPax)C4u6WCu)vvaE}gcPVZ%ely9T+L;#yV?6o!f~7D zdf^GqD6g%BEaLBP811bq9)Htq8f_Fwy=#Bw+p!^N=UWghj7GjA(%)ToPE zd^WD?=A4^=W@H+9&vL4(Qa}~{L_J2Zj0Zl0fD`N;Y$Dt%@Wi#cT!Q?kU zp4^zzqh=+*b>$^V~T!*3qHDqF3qt@I%SQ>Rg* z&>C!LJ@$fipTIxrxE^fEM8L?10nukp)N2}?fuNBvJjXxrm?;#Xh9cs7l~%Ghp2zh- zCgxRmJvY6n)s5Y0-@zLxchpTF>Z9G>sCy|6vqFRyU7OQs!aAi(ftXE*6BM@vWi4(S ziyYc{azoC#Ugo5vI5;al!%5Z&rGrkwvLS|E0=}vD>+p%6QOXjR4+3lfE3`Lb`f+RZ5Ev}i~)yoO5^P-b?!|W3s`Qm(rs7MR9PAL_ik)Ar4}}y zbu-y({MU}h<=6GtIFJcWj8uWrORj%quEhrFALnR@INWNH}4#l7@35FSiOsDbFIv9_3xI%VnDflW@1FnsFjWUHc_^b=NE$vWl7s?uOyFIJ4qoVZY+(01XaX6v{$>I2> zQsnr4>|i-{K;Qp*n>NLGnx5+`&9M6NFsNymX}n=kG>al^h7h8%P$8GHV2x+5Y(6L3 z;ySmWv*X4F(Ej}E{;8eowlF5MYJ+;dJI*)<7)c5er&V0W>{`|X5Jak6#8Jp{y3;Ic ziOC!OGIfeiD#)}DmAiKC8Sb1Jqi2x8E`qmq)bdY)N^i_vKWNUf81vF6U&*UGD;34YB$-g2&b- zyco6mIfg~a6{m;?j*UN;V*}0LP9h@6B1>$noes+ zekdtz{PoCuwLe{jgl!dz{@0~RyfEY#dpOS0QIp@r$zr4or!&Ext6+P!nv6Fzb3v7^4E*iB3u%k< zc>XnxZGUv(_K0FSCE=g>n5Qs?-g3oXVgh@#0#dDJabRslm{qy*xDHzmRhSnE4ZXa@ zXIecitUqQSU$a~(4+I{R#m@ulVAzTZe7q*Vpp&>LD0Y!SF`W`_i(IpKqVuo(5aw^T zVnF)_@Z-H#FOi#bsMkjx-Vgdu!<5=fUH8zM4z4FAmKF^!fE*VeKOe%uvu9VGG6ZcV z0=MXXVffyCe3{X^3%feS*gP9|cO&@Ts1!#H!ZU z+jisx8`7UL%F(AyhvkidMIr(TY-3BQsaBOKYLDGLPFGD?Y~qHLXwkHPulYaY;O!WY z_C9sJAS!zQB|jkTBKoGxde=a37p$|dX^aSh2fKigiXAlE(+)<3%PLw9?fHU2)DWwV zD~YkYGJaeT12a8vaq1S@xwO*P_U-4aT}CK zymYj}m(JgE`hhR=I;^v);Kz5_WS1Y7ips}F_o8mRbn1*UHnTeV&QPw?dhmviT`lW( zx_raPhqLydTXO>Vj$d5z7_e(2yb_%~)U(V@N7=I&T;U ztyfENM4klMJFS|QGBh__@AM+BgNwdpxZSAr+Pd3tNBkTdGIf!`&7@uGlA0YQ z{qVa2U#n^UL3Q7YQ-lcaL5*0LbP?@s-cGDK?BWoHj4ji)yDvsc{kFnpk3`nohM)LH znRJ#+1D0te?zIvIn7pg^$LTGIk)BpvC|qkJ#AZ= zPeifM7`944?Q6BMgUGuG_DO;Of-)CBXLkBJ=U9;Yhe>YUg8}dJD8{mUlAhKNRQ4=H zYH;7r$Hb{RE?#;Cu>#s6)k#YXT+ovl^fxr-&NPs~w5*nm`<73ZUG^z-zq1K6_MI)E zuw!^(0)(H2gvD_!XpAOu*@bRQy)ksQJCBaWJ*f@HLy(g6h(%88??LqJoZP=^JYTq$ zJo;(dpA531y!DBP`g-SV)X4zA*uDk8*M_6H#0=c6A|(Wp2o@itoJ!QX4)1fYk2V>rSP9O7W9Q*zn%X`gRM+eC$0Xx^!Uiv zdzJ~53@)QlwJDOksjVx(i}wvN*_yi~f)^#FR%(N3!Ry|v1Me#CIH#d09`U2tiEytM zrQjD9Zmnp-8mBL~jz7YVBcHGzJ>FyMZpRE~!rbKb>-$1QqWGpP2QUaYd4gLW{Z!bDLX zcnfY*53f1*K}LEzH@Vgb;?djM3t+CfhvIx?IKfdCt6L+iQBixcPpnXJF%hS^LWn*H z!>Ikoh3}q}r3Qfq@0h<6qP5>#S1Y>ZBA@tzOr8QpC!;A01jc(om z5P&0-Oh4B}LaN#<@K#v(UH}g{=GA`=AB%U{NPAj~{uLzuxodxlz#{pFK;HT*&Z;hz;MN%md zO84>v2X;5&Dpo7sbQ=sn=Ib0tuen^~1>wJVJr|>O1<41Y13X34zJL1tOY6A9qjJHB zSCu;F!^DrM%!p5J?0{+#x}iU!ji14PpChF9TqgghZF5|7CbkP;h9}Fr zKLV(a$kDoT@un!AH8P~_9S^@Z>fSWqUPK$ofg>VKV~N8J9kMy*RI~=IE2#TN!(~A5fLOorgtZ z>xrvw?Dl7&3ud*a*VfyMr*4IgEaF?TNrFbQJE9GSATGCP#$N&o2E9 zD)ecCEY{rR^)7D})Q>m^F|JY~x#KY7>#2=C^z(6@Oek?XYmVZt0CzT<0x$3; zkG^mKjtX>))XGaSN_#oF`({lION0zMCj|B2d@xJ+)kn-O%gnATx zbol-;bG(`Jg6AWldof&Bg_m~c81?q~i)N=o4R#a8DSD7QmLsk|>R1Ms$lq3&TyYnk zz9;7$hX8I^JD=rW55gc;l-B3NRx>g8Jt_rbaD8oS_acq`1 zb00qZ1J1jkiP*inFTm!HyeIM*J0|O?0-p%B!yJd&whs?;M1VRqEa6yuYF|D^ls4R; zwL-QA69;{=+V2+1+^Ui)R!j}dYmTnjdB0g7uB#tx30IQ*_+4R(6tE?7kfyYtkv03d z2#kS%5V}?O{j}K2Wha}Ao03~At0<3EtYM(gVqUU$MkD4nydej`F(l`bkeftDyUaa}(#pj`%XtKjhEr04u*{cC9@S44uJQI$}(JZbS* zj8LjjkFxC0vC@n7_XYwyWSBr${vL1f?cYtj&DD|~NkpDxYC0O>3iPVe9s^bj#kWJPL!`J6R}w;7Y$Jiba0EK)k@aVdS zNOaH&xsSXP*`%fu{LeF!#`4pi(Y;UJIO)q(#F-Dtt36!KlrizlXF$Fq+55WmU%v;n z?SV$>j8%iS#sU(67P{3qd6*+&bi$dQIN#Hj)}ypjF~NScn{e^4>;vA9=1Qrkgr9iPg~$l+dk7rn7`Mle;(3*-!_x``n?%E*ore56p#IHzy0^m{(aHR z4ilzm?fJ0e^6x$MzrOe>2dg*1Q6}6^JArM)_1B+qQvN&{65_PxOOyY{jQlf}{~FpR zEqpq5&JxWbZE^(Y@7ey#8T_ArE>d;>^r-ZXToV3Y&*k55`N;$OA!qG0uBPX|{@;H; zgTFr6e1xYKZh2wa0d{RN0|@c|-)ZzYeh>8)9v4Fx$p7a7{oliA zA`^T!0%F9)m;K)-?4JQtD)^8l9P5)c#{B1Myx(Na?=w)WM z_S2M<#HFX7h*y}X{I3HiK7-E9UIxM4KSKzHhlPV9hpsoU&*1*=HA{(73JS^L$&6Au zYp-uwOq(K6pcAeh?bAQ1z}qez;^G!A20k!PVmu@*8IDVeBR-_Ls9fM}4E`U^Je?Fe z1vIksqJPf-*ZcD;I+6kr8JSQ_3i1wvUoJsGVH6~a^qo)`Y-`JV%mUfHa7>4-a(G65 z3gjMSOnXZ4?Va~_mFX1FtN13sr@v2z*Fz?(zpqX3h^}0^o_YU}Eb}P-6hvhfdO*R) zP*9CF==MJhJzEiak!e4amnZz^EH|~tFh1!h_xAPfIjDxg^H0mR_kt(U&HQ z#<8jt$_4DIgvxxh(nlXDc;$HMu&^Nv(3D$DeD@bEk%21;-!geY6h$xTPfkLUMJL|` znXeQ<^5Fs^B(yw=9IL#%yt@kZ)+ zN}D+iXf5(DtAZ|i9csu+8|Nx(V8g|!`#?xmR+c6>;DSh|^bOQ9dU$w1({KrslQ`?+ zlnbPXJG~)^^JVJ8*ChaM`Ycy(#x_RZ3Efu7CPGf>odLPM?fx&z)m|B6X*?3QXR8OT zk;bF{J_T#3J``U*_pWg6FBJ6*{C&^+APKMcD9_I1>-CA(Ixj|*`SoLDPY}w+=3iwh z8`(?L+`C9VwtTr!+Ez{#lYhacJuJaSHs7WY8jm3x5t%_lPsmrJBlnzs`$h>@z0@^4 z$!{VSiVQuD2?XBLLOs~uq7tLS@l=i_P>M=ubXD|ookenfiA5wR@05^1r3);M_Iw9W zYuV=wamPFlc(u1psJI*}ry!H0N+CL3ONCL006(VREIE?QbXL6y?T=%$wCQ^y=`0yc zcO~MkMF2XDH5WBhnQ@3jUQ3ln$4B$AC^(Eu70{@9`g5mw zh1cSWl_JWtlG03*GZUA`z2d%{j*~4c#(~KsSTB{`#!F1*0{?N@Br^#bm(;I`r2BXF zDs^F@NWcVLp2pCkM*^79bh}|b?DK7UG0jZ`*^!hoAYE^n7Regw%N+nju%)Do`@pF*!w7F7mu@4Ee_fbg%z0s!enQw{ib5s?8r@c75K2%g0V>>^9td5@YKf#-I&XOSJ;hqGy2A(uA$tp)j;ymNv>Xt-vY$l zCA5Bie&nR2y(P7sv{}SQP6HTv6?TcVT-g221#Xdew2L2k?aTs&h*kIe1B9N-wQBtO zCaH+L6F@n(>lc<|md)Pj9aOAmPxFf^;|B*V_6Mvs9LWNB0z%L_m9peNz;T8&jwH%O z_88;H%2~bPC5-G-=eI9E4RZ=)cs!$dR4rg!YJp8BA1*dQ$jvp#CVgc9W+zu z(pSmP(~mfO%@EV|pQlAP>oh2KA%6@h7btHeP<;2uYZQiOnOgn`qq0!yiMCFGD)U^* zodK)*u{C`0lgv)Qv_V8v+*$-C(bHV~<@1t>T|=}juk){ssnW#dF=)4(l&h8u+%GO6 z5l;Y%ZO~4Ivz(Mv>>a>UfWE>mg~w$zn3iXi$HZAj#fokH9%nQCQ_y zJ7j~$4aXq}Qh$z90?78&?1#eE3*3{Vpde(^%C@@~#5o{1*rAB(>-Xzhi=BbtGLi7?hzum|y6QWl)H-C9J@3=eZl-Wm1A7=B7qwv4! zJYf2ge0*`X?tXK_cP9E0&S-;eM~|s zd%yeR;l2+&-Tc7E*FWDWW2N<^779W%u#9YFRi7%*=InTLH*?xO%Gm<F@y}_W*X8D%}%bh=BB+Q4hSNCLPBQ%rP#6+^S?+6Q#TI}}h+Dyp2 zP&{}%gRC|32IRsDeR*mBb>nL&1>cow=<8*E`p-v#2AJK{d)4)-7gt~P6f?1xBYd5V^t< z_H!ooI7PL}ILKDjPdcFtqx*ny}_{py;P(?IZBr~S>XKi>s zj*OS8xc*`G+dqD;`yg)Sd^D#yQqs;04AHy4f=z+!pzk~7WppuW(kE{-4aem4IU<54 zQM}?pv*f2Txq$T*YGp~Ecz+6Z-h1ROIUVJaI-G3Glr^bV>S&@7aFZ|L94ILU<)5Pq zJpR&k93u1cAZ@9Ew!Jt)yV~9T@L)GX-Y=je9?`CSwb5W-^!&0a-FDR8-o9IqJeYF) z$(1Mt|LV)t@=YKO(r%SEejj@2(9|-Ls*lxzDNm7+j0LDGo6*pDuS0kt zGcvW;ixkH4aRa5ZCmtuREuBK^^xAnI7uL2bFVZOCJY~b98>k$cF!JdZ|gSMA39G=0NX;edD^PW`2@~gLMb8<2wVHRqC zwBERTbgy?B$HV0^uYZ|X!cs*!|JdhT5^?F~3Z5SY+>Pxur}MzR9C`U=sN|x;%kI(# z+#TZG5b82}cTPB)YLlHZto?lEljjHXmO-_1K=fwgUaCWcV4col z(I##aqwL;THR5)Az0SJN7!+yReisQ!*VBcC8opUQtL8;k6hMQy=4P&0)!(1_!ct;nbARgVOTbZ)Qb1PpUnOJZZi%e4hSbr}LgFuh zl4_^&hiZv0s=^aRS;-pnXo>Hnm7`xOd+%JO+~~yzlH;zD%rRqG9=YsQY}E69P4~>^ zo2k%b#M;0Tggo7X>DJw@F&rAbo~IF8>j_#zJuLIv5CepVdC2)91kP)i;?RR|oCT&IjEWEU-l>g=V9 z23={8Gi;S&-)_HWOzpw z`sTZz;G4*?qXGWA^vo{rF+SKDj7H~4!iSS^smr0A=+PhO0{M#%w9&u$+91CQ4e z4O?U1AGXJ8a!Y^3b?8xXOEK+hKhY==D~}??R-dj-u@^ZT8iY0 zEaK9Qyd$unr{)Y`6Bqa~Esj*toJ?Yl5Bq_);Dz(lX!-6;&-Hn@sK5+Lp2X#}nBYBh z{oxE^S!tES*JWz|{HH_`guU0xwgLaQ(D^@xt41t(MK z=aoOn>475;REzK)o)tL)GRbJ~U1qkpko*OaV-Mq_1(NPIYTQy*zDC(0Z3PQqB+hjQ z!N64zO1v{F-@-rey33d+lizb>6@KaIL-r@CtwZN0v~fF*^xUW6QJ5WjVs98m{E*k4 z{Ib^Bcikzx?CXzk{Djl>8m+{gC6yTq7@6K;; z7c!iC<`cVj4@ia(Fmr?#yUR1*NT+khlbStYs=iR&v<4zkyhxjJ(+8`u1hh!;)Xp3s zQzYKpx}Ym$aK3S*qM|x(Wrg?(!YO?}jH4L2+RuJ2Z1=`!MI?)Qx&~1JO3=P8g(&4+ zf5@nUl**4)70J)Se&B8%;0NcU>E5lOFuj4#qf$NaClNl(m?Mhl%w(}jJ!%H4h(~ow z92LicF)F5V;3rc~EN{nfVPPofdB@&XES5Rj*y9AG_%&gyL|plfXR^<(Wdd*`R)Su= zwuL7|Z*zKKxQl2}fcCni_I$jFFUs;*V{{;bFW!@mN#Q);aRtUo+_*Y{HguyudT%Y) zJZg1(K2^K8y~PX&64Aqru;zrPPS8A`^$=?2ijW<2bKoWtaH7{Qf~^@|I37moU%U+Z zy23-tN&2Cex7UdlI|wvHYxz_JbSUjU9gD4RZb56!3)LjW+8+8Klav=J_?!M~gPpMe>=f zuc<9Ho<~W|0IiAX0z`D*SR5Z^J*|idPuIN_`IWayC?g<4<$cR3_HAC+9wYUhlKh`A zB>!On2v>FK%2z&0Dl>{NgQ7mkV`P7KW{TSxt;l>0eKMsIz2eDp=fNXnM`F5!u&UcB zaCU}UD*G?kH(-vjlpnZ8T;l{VE+6=Ahx#sGf{9WHDO@Htc_JsF*-Ip82}7QX=LY-W z)HJdLTD6grgy7=2JQ=Fly7#VXl(Nq{>Wf8yR}8}J)-$S1J1O4oUHr9IHpk9CJ2Cwh zESK_6b!|XKe!`wln*HC$0DPB>DxZS7;aOhwij2RFhEmW;m9Vg(k?9=){#EI1Tva!Alen&0w zb;n>my^bUF2a82(KFBu8oCwoseqn6JH({xCY+ZF{vTM~|x=99){NUa9p59~1PwkZfEuS4T2)$=$V?1(s z*(_0A`F>`cD&}Is_l*THdnxdO>qRr7;%M}>BwF)8SKfO0mt2lB4>p|;GvA3V_?pjJ zH9ms`&BTb21|9RJuG(~E!hTSKpHvhfmw6?c;x=x;6mH#tqSp6kqiG+IZ{MD4&msP%jdB z{i1^A9W2uiraZq#dh>>rPAIDS#nUbAY%Ez1x2P{Fx0~;7!#NNs@YoDVks5B2LnJ_0 zL#)s8>dJK`WOcx4_0S|sVIkZ_J(AEi8AKU;rKh*ybPU<35jig(XzQ?Sv8%P*J9}J4 zBQd`xa6V%0{`m2m3y?IJ1Obj1unfIT)b1B23ETin1OX^w!mlQa3h2CMg+Jr@90>0N zlEr423I7sjp>ln;24?jD0(DZ+Ks@PR-lcCZ)s(!HG-&>SD?FH9Ddz^;-EEi+6)WN_d7zzrm2`J%@@NT5kG|J(lC>D)I;;y@Vg1evo{?J|q2WR{C4yezw)$ z3Ifw*czVtqnWXh$?_Q9(^1lI>lsy&w(iH?4HK>u)?;i52t~tn5WAujZ>sG|`rDSD4 z$A=-lPMo7KhAM=&1iT*o2hgT*)2or^ZWw)=n*NG>l8RC7Qojbd9nktXY}Sg4Uf&@4 zVYib6t=k4q6!^*~2`p4{b|6R>Pgykv#RLWEHvbfIKmGQgkS4!I7F_W?iKM+kzv-*H zbdHMi*0#RmP{FWDXRKXmJQ~6Hx33sMJBA!&^!ocZ%M2nB?v878+$Y4mqWqsA(DVa8 zbkTgMO-yYg?ZWOo^JLqjYs=p=hGDnw!s{m@B4UCK z$B!)0F)DX!e9GY|O}FjCyXM--EYTCyS=fvVj=ayCo^I^z%iT@7!DutGfg!?CH8;|x zWtwG$V^0@U8OMlu=UeZRQd21hqlsp(8VSJS;MeRojGB7tMi=MUXWd9w@ea2!n5nlZ z!CV)stK8_ShlPCwM<9Kfir8@S;?nuDZu;LT!W2Fxguh4rXlr>0H@gjx$huKSSo9&i zBfaqo5``244A0uSW7f=J>YWH?UE2BtO#)sGOM!g^_St$@`;IRjz;6tq1~7ocQnfN^ zD6Dq3NUUpUK%mKN`qgR>DsDc{FBWtW7fzQzx%(6Q<+?K;Gy>E0-mV7DwD-hK}xRYca9M|bHBk>L?Y6>dWjuH$7h>~PYi23v_#pL z&;V@qCYPl}(5pa4mn#0d2*;mqlD2f;ipWdz29r4SM0l9w4MxPGdRr%Q0U*VQ9Kguu z9ZdMm&`!e#X)w<#R6lBH)#(%L6KETgYFPYVQ4Xq++D|pgG=xR|m5M?PgF_XQEB?A~ z8*?T<;mu%hO`qgm#dFGU?UpYE2ET$+wc6=Njzy6bHNFzfD-`o(&xv}>BQkv#kZFB$ zP{bQDgE@!Eom+eROH#HzY5FzWOSw*sa5yYb^k}9b?G2R)w5vJdVal(Mx<`jKlOw;N zWM*fPdy0Arx^Q)}W-HW;=SHX76I`K>jblO8BFrvO@nB9Wl@6U@*VV5_1Q{0hvhey? zKyXtRUW75*uUBgAto`XF7~04D2g4~b z(FcAAv}r5n9Wbe^oh?fEnBa?@Y$T(%PN4pY$s%vx4_Z#LBwLUz`k%h7go^7;>WWIPCOo-j0R7sF$z9Cdi>s*zoJqWiJeeiBl<-?xl}qWyB*S&)|BK{2QURDgiJv;yFHVa16=^RgE!Od;SfB56M`#9*HC z(%3)yCMcNwCKS7;7Xcni-xALUW8eVT9r9+JlKELd8m(uuv;<4P(kSzJH!WceKYNKbbM*Bl##nq59wcqwg!<_>?igF{s0L*zm~#5w zS5^dX7O5xArr&k*I1kriL1Ru>>qzZsZ4RbI!lHWAt>PC$|86h(Ghem-HvRW)3%QTd z@W-3B8;{aYsw~&vUlYvKEKKH*V@Xg+(VJYPqAI#(K8r<_Ea#ag{G{g%rKKw=4O4A! zh%rx!Qr5yN*&C^48&_XUNq#p;rz%hOv3KXX6U4zM0M)?w`mz(_qb{l(x)re$b&)^{ZzQDImw(M1pX8vlxQ(xM6Sjhx>vZf$Vd2pEeHDvt zW_EYx!phEaINa`HNc`gb7UwK(?sLw$Y~*iH<;o4O`t^nJ3ma4D<~<% z;H0$6qAVepa-_Eoy-{NCgyn4pSl|1m;6Vtjq=PnkHFF(eYRdqPBfG;blCEm-`u>-B%#sN{;j;M?ES z*)7QZ9i8q^=@#2#xOR@n(iq8?abyrKQOYl!T(cwYcUw{8OKkp-y+40^TW!ZP15PI~ zZzI&CshvxiV3*vF*)ZV8wPMTY)PqKzxExq$JyWk)K<%Dsz6H(S<+TlGz=_B~BZ;^# z6Zf-3bi0hJNqO=q=LWTBVpEzedQe~>$}i%UjtWoCGel70Wsmz=>A5G-vh~ar@S_Nl zh&B~5Sq*{Ao;G-ixViUTM|!Nh{(nNHo>kjz zU}zkKJuFql1xrXnct~p3UD!n>pj8xUE?JMTs4p^?+3u#t>7_&DAluI+yxNe7|E_$3 zg!2~Ki{$E^(L%`q>03YE&g1G7!1#2gR~MvF`M}s6T;Y7p_SPSvvP@mw`fy&xoiUfb z9S3r@kMG!APn`CwJ`|5#{=14|G%4>FwoNb^UNK*Bci;VKp1gJygYGZyOVMxBy;*zU z-HS&p5}kE?n=Y~)fk#DJ^-6l1PH^UZ()s)i9;|65+V-xzi#l;c30Yq^JW^iu1(6rE zNrH2o!Fz!WVs!q?5Ivo0(;D8J-#`4)Rr}7`WllRyB9vr_BYHPn&*0cT6I}{M60!?3 zmY8%vJ03tZP`7C+Op!OA&=jUt9L>{Ep`n9lR<|+FeOBX5+fa8O@&C$Dc`elq5~GfQ zJ(+3=*9mK*rcrIOa7vqnqQ{jQBvyAikUpa$5;Mz_|Ze! zL)rL3(f5&8NjR`UW$W%*>se}$DEyNG4^x+hQ|ZKbd!UH#lHa?EA$!;-izlbLS;CAv zV157+v+ax_S8)JQ0vmVsHCo{Lf%6`2fYL1Rp4m=RbG1>>w+6ar-gQAT-^aWdL_qdaG>94~GIaVy|U|zc~%EI`7L?B6UA)SH8=u{|J9UaL3G_Vzm>5YIhBM*Wm`$ z(zrdSv854`6N)l%EwBggxUM>QTm-!zUW#68y|u~~-W*T;L8N>=H>`4b1zYAk8^Uvj z;lfJ-k1q_0gj}=LS&wy-?N==7&mPC*Rd@zRwKBLQdGJ5TwAE?|c5<{+Z5}>8hclH# zd3l5R^qOqzUS4o}F-PRHZY$7+-w2gU%@($Hn9_9?HSRUqtQLyForsJQ!Bl5l?vc;e zcvn@LZf5sDBQg{el*4`KI%v@rZ56}^?<(IBL08F^0UIAFec5`H{@WM#v!QGfk?S+~ z5Mv@wy8>e0=LDqJZ{2rp)E!UxJrq{zt>}21&*smBcHI5XrcB;fmiH;OHCLV^ZQd!a zyWO0iF<+jxfs$%`+<%eIVQDn`{gb&soCU8V9nk7I^b@MnrCO@j6yq?}zVtB^U7;xE zVc>tZ%zxja9551!-Sk)L%m;GcFYK>KNxlf(T3sh49!L+B&hkr})=HLDtrSmbONW&D zEtWS;AkN<^?d`#!mWEyWmR9ncWp3XXQ`L}b?h3jYxq`CH;(mS>a%9wVSo-bItU)7U zBLB52d|%!)pO7oLv#pacPDz!H<+-NE_vQ|Y{%H%qb+O> zmfIH3h$rv0XI z151w#oS^vHzi6nM#D6z@SU<&e2%{B)12R;tuo2l$jh8E*4&AEvvnSyQAvS}5k`i)kDx&(p6iE3fb- zv#E^|>0AkP_C``r%NDlkJf=I{`2|`Q(ngg)=WU!*4Q1^b({Nu$$9ou`!k3YJHF}Dk zd~eD~Tpcm|!J)T`#&860Ma~w+=m4%`qDpi^?Cf%KSG|ReyDL=qWce8rJ?n_r2j zwgOC^-7tfcd{TNZh_e>l+cJ_j?LbMK`D}!3qnK_-_+@%o{-**tihLNs7v8sW6a!@6 zj15s13nt`fF)f%_EisXeTD(~Vy3Nv?6YFt@hfG>4`HBk<-M^oj57sqi`hAdZ#QHgG zVA^05agMeu=YI&03JAR|s|FYhogna6cv6o0cmgISMm&{#71lWv(Ad2K{-7D6H~7P= zdW(=%9St`P=mMe;1zNyd%ENqtww*rZcybI7uv0mPwTK6Uj$3bw6z(fff6LULlIz+P z0JidQ?kdp8_!^%-N3sU4iASKeq-et~iJ@W-+-W0h6=KAmbP*fM2 z2by2FP{o3+wk_|D8m(x8F8r%XR(OcOJMp&1V+<~bhXe$re7&p9!lj*tmPRAl)O%R2 z2m-Y)yW{fQH8mud5$3S**;Pkv$na~V4WwrHHr1GMFtD)FQzlaz=~~SQmz*$D?qn%T zPj1t$oVSHIT(-c7NZbUIsYq1qT?8=45_i{{fqmF4d&1i5whL)T(Q{)Y%+gn=WU;VL zWp)cCqmnuv2uCzu1~Od6=4+pCF--C$kK3xldNR&4#h%UvbN4rZ+n2CMMOG-)* zRMeYgv{E$?e##m9NxxXGBILG{C(0b-UVH;B?8x$-(0HZNXH_lyhdZG0AF8`W<-%t( zp*NWRVZS>Cl6MBBw1l3VgmSOkPDLs6ZHv+lTyTs+_Au-#Jj1OP=AY9DsSx>+2jU;PpkT@Z6BBFYaZuaN9d zmqnS-zT`1GI@q;YZGTjcXv$R5fJd--Myqr{ZTO9!3?hdnc~3z>A?b}^e?7BRI?miu zV~0S;;6fr6&5-viW4-f2 zjSW9tY_x^ZV%YzaX+Q#z58qG&r_|<}!ykERsv`YBVLBVR+0Gd0UXK!I75k05didTV zFYFpQ()hXEC0DF;US5m>(Vr9tx+TMTd0zZ!u}1sqXqiFh;~l*%mgetsoEI;KAb)-~ z?TUo1uIIPH6atUbu()ugDklwTopeSVCatK32E@H+`(@+>H=CrD7Gpgb(S}DnQV{Js z#X2*l2hA^KVbx+&15rgbat3u5HJFmY0%SzA^yxj_dkpD~A#{x0PzaEi^RmE$037>Y z`J^Jn!nLGOTQB9yoq4sZqq%iRhQ_NTX1g20vzusHF1IyP<Q1l#F&SCQV$MY6Hyjs(;E{JS~;mO)40anfa*+@b(#vZ-aU%Ehy+I zRW>i`ff{GIRb#mkg6Ajm2TQ~I%?s;()Pxo#RaGWgSuJ}hlOl4eW=GBn@6X?y5du2q zSdDT_dJy~+z;6q&gk_4*vjEm_cd>6^e!aA_Sm(KV z5*dB(f6X!!yl;d@QbHpqsANs9P~&usjcJYitAdL#m~vvmV_ww-$*HPT=`&MCMHPuv zz3F!G7m7GjOa-#u_DfMoIP{t9i^f>_sepg&bYO%s6nM7q7~#T7vRC0h++X3} z0s&P@*=%ob?`{>n#9zCi5!8kQsf`c2kn{@WKkdhhWj#`x`?G|xz+kZC`$U#II4W79 z(FCH23CiK5L~#qS&t>mpBdgJ{{r3-mJKhETn)OVXiq2>{FIc!`D!kyALNT?;AwJdP znZnlaW3->ZV@5sf%a^##wo-px^iL=aIOy>gv^WD#1`6X(8NXpMG zf8CF0Vai9;sPT!3T25#PgmZyMPL+|~?|ZB~xh)Cq2|OBW>+3zTU=C7OZF%`DdY!uW zR|m@mj#o5N8S|BH!DWb$il0Aw)8AO9sm+w^Dr{y+p`(lyx1Dv-%5}*Cu7O42Mk;7H zf2LFamxHs99q0sjnN>tMskb+GQgbYu6f|6tGKI{ps96NK!UteyCt6aPkugKt)cE}P zn>gx711{e!I$TV8BPB-Kg%o*1n3Uhepc;_hdB~N(UDKws5?4p_KkK%;{QprUA}ELU z$NIXNY{55O*lMd$ZWbMDb-v9B@ z6CV{&z`Vry`tc9>$iGWo{&wv6O98V~_Ig43zjy9`ziA6WKmmiVNzNYmS4aP+m%JGI zE0^B<*^KG$3;qAN*MIfZ28ds}C#%TW_kX*&|MJm_gY)F`PG^{A?GcxP}Ych9$NRQfLEe^7P{W8b7Q2(Dc|`F6?5wG5b~$ zg`jI=4vl|UHh?}WD)GY2ct=5GC8>bF?g3A<-~oN(>9@XICMwuUz&F`D(Jc~2$VdJO z&4A{8<22T1(|>9v&4?G!)EBv#zn*VLZ?GKI03JD1!ZFttsX$BqjkD+;w=Yz&-bPpOCd8N;ye`@jn zV{Osghvto)!LSh|g(l|BAQ7aXq6`W^&(w``brxw1finn{^*@PO@)O{DF2MAEy}7?n zmfPJ$0ceHMvGkKnQ43Vs+foEPw8Mz{Q(xfbl^N*wNX*sQ?<;YkxzY^A(NW=y=tLj_ ze`58{NPw16!>GW@l8=3jC0|r?7lZEc{Nhn~*jd)%k96#R%otv#K7vVjC2qUpqFnhl zf~g9mH#M+|5bz+99rQPfPUS&)*U{DSwamT+F2^2%Y0icvu6zbTEoW=zBkX>g2$6!r zNo<=lSncpFxOPnvGQ{RH%WiAi+fyFAuF%kHjy?bH(-p9^fI{F%zAbM~j8zXTz!S;B zwX1-LK;O|?LK3r*wnROJH9{lYV0O`R*YUBj;D}~Oy%G{$OG-(_1zTG;qGy2S3KjCn zSy<#{2n4n&EXsiCpZW><I58Zs-Jc38SnpV9w$bvdudiR6-)C>E zb+nX}2#g27wUMf@@m%S_j82gpz(T#s93Zl@nKlvd+s)Fi0J4VN<#n;!gXq&=RQ+A| z^^dd;Yq;R!RL6}Dujd@O_3r?R-in@u?(&Pf0cckGv9$RN2I8Xm-R`BDuK*)JAXx2g zX{_2D6&Tjz9I4o%vxn-@ zzNPDsNTAHY;Hh4({#}jb;ThGGz+(fMc1NYWs*8r0_nJ$Y-&hG-0pveWFZxszw8GH{ZtOBo#ojWmwx+rDI zrbodCF%1=%_OFTB^lD}2WSVRXDP7+mx6x6_5^!8M+kTS(-h3JF`6PyOs`c`5%Vk~& z0iQZho1FCd>At+>s@*yYm6*xx9_h!mh`c;Nxvr7e8h#upL1b+Z%NAQ=xWs3VvE3dS zm}e_Cn6F=%kNpV=zu^a1NH`WA_d=+h_gc;!X&1^X6{_aigl$%pZs08y7*OG#?he4Y zUVLp5vqVb!j}85A>%68OwAo95l#-k){1c?lm3g)XB z2C|U=>She!y(oRG;XH6|d#e)AXaV$!lL z5+~hbnw~>{mYD&1IGI&E&FKKm*3E$xHjE%z8iy&UK0MsB{gdfTNhDsa$==oLeN@K% zG&wSZ-E!%RqGx2{*jd}k@tdYQ=edHxi9rolH@A?bi$;gzMaf_%)kfzfu|iZep(2&C zf_mI-x3wGV;qKYfgT(48w|ixV@3{NSwS31GOD=F_)$#GI7R}q~OE%}BOLmV#@CMT1 z=OgHz6@ZT)Vbjf?Qo8{SbFJ};+p<{Gju|ADgZJ8oI6yK=W!gTGyHTyzqOF z4YLt6z|J+hwte?JT!X9b>CMTSVM#1BTL)yQv=Bjcn`Cs#F{4x-g*}<|E3P6*+AAOu zej#%3VQ77h^5J}4PwndIs@t^`z!WmLEK*W`<^96%b}JNL^0}>k_d$%!L{TM8Lg}I{ z!TqX-+$|&=NURY~mys0TdVs(%8>$4i^?P!hwYNTb|Ay zhv&T_!V^fA5k&8)L!X!Z!_SVv=9Q4i*C=oi?axmd>g?ukfGIhvwNt&`m@GFYV0I^x zY@^B9eshK*m@OEVxL-aPdhj?Uj4M|1C zm>2|xT$lnB0VkcknL>*4mW!;i?B@(1um9V;;E=a*N4Zz^%7;rcJW|=Z6q$^qMn|mh-;GB3e#%?7q$}qptDh~r zcF%6kT0MPIb3`aW>WZ&v&&qjUF|a#c+es$AT)kAi?*7xd+jU5s7LTKeays=-l$Xaf z;maPf!5+V;3mrT~zUAe+#(oYWvo42I)^~SrVpcVx~4+RWhRzUp)@~W~2q3OzO@4OttwP*VGps3l(B@u>N{GLnQ zQFIfRuB)r7r?*!FQ3j~H;?RSNcf=kW59oK~z3q<{mD-+njr3|2sdZ#VMQ$or-Pd%ndzl)?>`VsdfO=iA5O&^Pv4QFFd zR@g}J_MC21z-hHunrGW$AL_{7JINGL@m=b!o2xeb{CVE7+OWTRw4H=a$*E;gRmPlk zj~l~QEZF1wnHt4jrvvRVbvHT&hCMD5ldN8BH%-?$1r=5D0ttdp!rA31+l&EQ^|b$( zLICLghy0Dj%e$2=|Z0miU4h7Bv!6|f0TTEniA@z(r2tt=ridK z)OZqM+)vk_{I@8P1Juzgde$EQTdOwU{p!dzb~dOfj7OD0>~NLaAul!y(hjDT3e`m| z`5mtov3g`uI4$bSKFn1)bLA)oLPNhySfC%Ug^x2K^Va2JV8Dt#82((jK$K_0+6mB@ zKi3l>!k@#c-FuM{`m&s7u)S(n_l}UYg7@mEPELcs7f21%o#W*rYFju*H04Vh%OJQDyr&u^0^ak(`uvTnKmQi+(&wkWZi&y$dn=5F{am(~bg zZ0(nC*1~VOpS)?>*4}^1Za&AD%wZWb5L+a1*kn&mK_U7pc&Ij$VRds^De9?Ym-=8j z;gHD~#vMk${|0rAMI$tcZ4&%Q0~8~Q1rn1xEbz2c6UFP=)1n*}iy5sIUESRVK*UmI zafxS~$6H5YrzKqnXg0Z5Fpr=%Uh>?mc#PM%xj^9H!DC`WbdwcT>>2&9UJ#`)bli$&FW8X?lF2SB%AUc{J` zW`QIUU}eYQaQx=^V3DE^(SBl>FBkAIKriPFOLMxAQ;YuD?4)MEPL`8e$Ua)DyEZ?VQOH(0Y@*nlJUHfWya5jI@+2hWa%IunH}%a zkY77#m@lHhw^5wf85FSI{b-}H(zbTIO@&JHNlasin}DFVja2h@%KN`9Du0^b{5tSj zll=v7Mv?|-K1ToAfEV#<3FyH3iz``(QZGO$7imhOuVHLn>?+h16OPOjYN%voXHjr- z5B$9HR#mGu{0Lx?73l8hwn~J;N5427n^Ra+vV5lrSGvM5PY@7QdNUS>jY{=~Yz)+$ z#qe43Q$fn%lmzD&yVPGp`QkU<4yqK8xKlj*+T5tcHDa@`x}R1!1y+ATFNdMz760F+ zap3l?-e*XNQoo~aQ4u}7+$gZ-Djh6;5v%ZcROfc+|53io70l&+&;a1Wid``?^34v1 z6wK5TPfPdZcE{s#>GFmEpg0nJNbygN^T&AraS>${hLF?trA7d$ENk|uy|e#r_O;}- zalQ5Z)uD~c%JMuS0zS3y+)rRT@?S^rUnd1gQLW(75VkiO_f%y}t zv!=<|qZJd1&!HJvO#AMgqN}TPUq41Kp-M;Xf0^a~79~f>LNbyG<8DK|MZi|&k$BjL z-Uq|UG6%`xK|TL<_5XlHe*emI5W|WKZD8|jrPptlvXCOt8lM=m3r?;}AO16K|3CW9 zukO=D62XYQWT1TVXsGQsra3M7J2+b=@?tI8*=vuZFiGY4>yuy@?F~-xehP=iKQCKh z@T&KH6~pr)dHU2#28NXx$0e(GL)5}tTcf`}evhXhw`O|Gjc>e(gcS`uJ?u)feW{jtj zCpyi~eL8~yQ7IdnsQ0P-G&qbY@slx_IOc50qW_3H1Aa#N@$t|wu`v}DR`4`R>N2UE zwD4cyr2=SQlLm2RmMzzAMEf{?!4w4KbSY-Mt36J~p_G}w%^QZgk+0Va^kG?QZHfZ8;{#XWLNeEgV#l?R{^>Lw@ z1Mz(t$MW+Oii|c?M^Q+EK)0E_Zojudn4ojh07UMhpL7uc2*8joJZ&23zPQ?LeF z0B`w)3*z0){+Nj)&W4L6c3zDC*TZ!5z;z%>J(g=n2vLZTb0?ABXdXaTa{W_JW|Vp1 zX0oFpLX3YmUt%E#Hhm{r&|3dA(%iPki_R77`v-fVe) z9&ipK_L_{*mz0vBWvANO-KDt+?eLJsOqL$Np!ouMQ~zE*DhD%LXbd%@M5OI}IiSNs z3G*WK``vF38I?69(jm^ua|WwXdSKcKB0Nl{Tz$u|d7BG!Au8lj3yo0Ix8U2`@CQ9LJA8D$IfKrr|{uNix#fcb7y@}+l1K6 zAGv)6Q?TX;28;O#cFffK#;CN@0NVgc_|qer_pvkzGWI1d#m9Vrbr)PcTrs^r^uq(x zWP9gSJ4G1L)HAZ4r{8T!(#MbBYM$Wu<*w?ca7>puC&FNHhUDbrdLN# zFpi>8a6EKCjE&zGrZYahiLb5Gd)&s?j$V+8nG;(OL5L8(hwsOaBII*ps1>prQk3hq z(NJHLySqc6=Z_=r`FfUhZU{L92#m! zb|}G}tgKDbbprN(HsV!;|8_N1=mg5rioU94Q{NZdx4@|_H3sbBMfp1WymZwot-mD|JsNAN@xHZI++6l2A1o9Tls85 zB;)0V$oJynT&={2e{P&M(mi!b1!LvjV~+Zzy3Oh5 zl=tGPM!@q6KqDnT3wr|r1;yzpj@Feg;5G{&$-Nott`?S z7M;2dI*6A_(E^5gT}m*ps}07O2>T&03m`A;6ny!WLIFz4Y)X03?l@HN|JkFq;Z})5 zBEzR6`ovA(xMM?Q3(#{aMwAn5%EU|y0TYP9J~_v2LmVx3Am_5|x^I4ZeLZ^F7UuAp z%O(aeS(>p>=wH=c0)MGLqh);6zAl5y{tyPqm6e~L@9Xbhv;x8y`NIofq$`7d-$fm% z1rYw3#x2-Z|MbJriDEZj*fPYKVGQP1s@7rC0=SQiMvD0?FXffW)U||#r5YXXZO5AW zf?~-z$rVR>)1nI$s3`dOhMVdMeM;!QRd-x*c||#Cud2zNPl$otl`%IS6YckdEu?Er zrVO?Yj>+liOMc-0+ySfXz_!LAXQX58?nOp2=F6sn)+W@ZKH-yN(JA%s)YyBxetPnf zuXX--dU669`}ld1_0*TUFeJd2|2bv!ZSe}Glk&zThp2=^;FCG1@^j}E;-)LRO8K{Q zrWR@x{tHkYEWJWr$QEkZH%OI=UoymXSza~Wk6=K`v6&e_OjB_el=CL}7=F3>^$(k$ z!X+^bD%t~a^?778Q4R*SKTt0hGHjIC0yU8+do4*Guvvz;T!-uv4Xf_!-@CV6GpN@@ zSU;wH+-9m?1U75iI(7P!;FlY%1Bb(4ewy}CV-?2;I>uc8;2YnMXO)atkE-|L{3U7 z2X<1kJ)TdMeF+szhKdU_W*uTAyy<#?UZ zxy07MgXtJB9(Sd)hgkr@hET&9Q`ewS0$L1emL`FQXC&o(*sf=CX># zabOJC)Lj1_+?%?Q=p?v{M5MY$qVHw>^!-D@L8hm-&3s};nqX1|LQKQQWHz$n7I1fy zX^o^#qZC$=d{3_dxcWpF(yH$YVGko727Z}Sx8q*1S%$DV=QU=v)hRkH-~+kjEQ?)q za}yyuRg^{^nq_=5XLaVZ*kteXxhF&vH!I-{f_VxijY1*&Xmq};u&uof|CX<_XAn3Q z%-=G~m-e+@y_Uu3a(>w5Y%zN&)H0xe1nn`^kS`zSapd*+zU5@IEU{@_6#+c-8zuPM zrM=cZZ~#X*r}Y|87v6HLrrq$zjbwD;2eoacZpA|Rv4zK#Ou&L*yd#|dXSiR9gJ_v{ zH6_yXPw}&}vo|51A5FK+@Xi-h2S&1Ak^sf9vsW5eU);B!FTWr{!|Jv0pwE#`_qw}q z(a62sTR0ZB5AmJv>_>>fx*bYkjd#c63Su!F_=Xgnw zHFjUK9+oJOH<~l1=@rAIBUFw&QG?_dwd%Z%=XNvJ8^dY6gVuFaR?n|HlG6C)M?~g3 z(<9zkoHDwy<88p;hMWZpk_hd~LMEb8!NEGa`Eq~C}k1INr%m+Z7qYVD6Xam%soaD;Ro}=w62f$(4P<#B6)(Qp(vu*9_iF{evFZraz z@Rv{q|x8S3Z9A|!|?n1=Uf-^w5p;Q2p8|cW6tzFz0 zQEkZoIui9pRLc48hDO7YPcA?5oiWS6dTKZV7UTC76jEiFiX+fVl(=ZW&+zu1RBOhm zYX?u?-de4Ey0GOvLY>JrJg12ra?4Sx`>_i_8n zoLZ}V%X=E7nT!2rqSmRVRNDCLN0Cy6srM3Ek^F9Aw5kPgn}a$Lr3=D){uv!EF>&qZ z+12`e5t5BKliQxQ44?{TENW2Tc$G^VD*R8hs<4>YXgZh}nc#P6(^zF`SiH%NHgj|} z7kr^EOdd}Udl61X>u$R{Z9R`6hiQS?*OkRaQ07p!kI}ImYN;rut;TzEPIHCI>lLqQ+1CPgwiTb zn9q-A+XycYmOpOhm**B=1uc1XS41Q&jc0{{w_8yr%SH0b99;50Cd-(Y%hxt7ImaXB zH9>ACk%?PerVS_3q4b2O3m)QOV)o6D@Kb!iyt)b-)NAh$(}bn&tLgZRXii%5Rldq* z0JS%~wRhA;HO+;$xhd70K)~ty-D2_zn-j{e5wZgBijzPh%yn}1<<`3`aLcnC_CbIP zde*VA`0;xJmG`k_nbyXwmecSLe?&A0g2Kp5>YM$k7r!+DtO}BitawOG+edI}w zEpEh1jy9!|88`#`ndvV6X$RP_U!}&1luP}!@PDZBviR48 z+9y?f*4hE z+Ea$|A-CP##V!b}Sgh?6OWv*!vMTcx$bu`R*RSNs3J^)K=F6(=Cf9$Fv{&eQkb-+L_BvCs|^gH^-P@bNiHIHsGEc4rH#ZF)JbX5P}`##rogSZ3bj zq4eJ&;g7`@s}$HYvRp*CnIvzP<5u_a=t%U+`iO-Z^Lez`@6L9n4DIsL&MKMCD2$|V zg}(c;X!psH7ioWOK3#sFuW?yowoOm`ZZG*bq@@F>vVnXcEs$pN&5h5lqVl7muO}TO zh~A(->EmOB#o+f#CH+Wlzy^r%t`As&V&DjrUddyjIYFkM0lmA||m*$?EJmF{6+ zmhxZcg^wydm&qO0Y|xkyb^#Tc*8LrNd&KvNJ)T6*w}!Jv`EL`HpPS>vdjztiuu@Pq zss=l{{gP_;#uw*UXAcJrOs4YqV(FEV7Lxa=347H$AAL|}Y`0yNsBp!0@6xXkv9b#( zfl4DaR{W{gJo)oWShUJ{Knb@Zbo~vq$I6C@2k&RFGq5|_k%#dh%i9i{5%vqlTlfSY z5wdhb$m6V9DnbuSU4OKE`hfqfj~-ztM)9FnW}`FWlEDCT2Fr{dPmH8@q$VQpEQQccW$G15K9_SSa|`EOMwhA8=WwrHDQ*YSaQA^^-On#oYIMgw`>dq# zxfFaXuK%+0qa1D&Zy=dDvy@ed)lu0}xkxclEEKm}S-9>Y02y~^<;Nyhy82!Y=-7UD z(#zps_LF1O$7H~1^zh_kS^HqOGL>(c3cKaJU~GdMpZ%GWQrfcHy}}g%*X~7Ov{dCw zmDoTi$0@m#cT*g?`)18`j*mV)alUwb^T_{VQ*PaBxWs9sntlmNwsjs=+V5jAhj| z;T{Vz?AZBVmb-_Ym;sf#wOZ4sc#fvckD%*lqBBWopm_^&lVq`Mr@!-#TqCy|;jtAQ z*Dy|e^>{LfF~QKiogLGgY>;;yodU7TV((p3?V*B#_d;$GH*Kz zJIYLNTx$NJLF`-YjnSr5v))Jon>54qT3f*z*GCJRcFdC8G=8`5qYh@YiX;M|1Z+t~ zDkyR%4=0b;@2tC=l6s&@#@r#HWasG|V3E6pS(L-H+Uz2?tnj}@zO$IuRO3vFb$IBWN)8~5+4Ngi$)TK5z1tg2p=IwppmD_2t)OkBwJ$qRX zwcH}$xnZd$QWWD5F1E)r>TGp>Dst0(UTntC@otE88u>&6V|~3SqE=zJh&U0%B0O!eo7=@I}?&8zybKjkaXsi@32|0B>EOyWLtF`(z zne>(|`3oa%KAG&GnmU-^9$&DTChA%7Ii;6dw>H(1=|`kzAHOr2d3mWQL}cY-M8tA= zZM-O(9NeBCG|_~V9KwBYx=uCOriI_!$^9|uo&Ef@&Gb>i;EH~MUN9zuu=JBQFjrza zs*e^z!aNttWwR4G#sJaLZD`Cl(J}EltjEB*C1=5jX!DZf>&$lc} zxSdJxYrB{J6th3d+m-G^h%STWC_^-bHYwjgcL{flj>hi?UYf)Q#kvxxb0=bfxXo3= zNR9nGffW!<=tb(l^a!a60v!PcFcD=I_v2(qLM{WYVsT%Vd?(s~Zym03EwNwXI z_Ii&Nn7IixwTz;HVWw3>WvVk2nbOgFr1|CLp8T;kAAIV@YhB;5NRS{Qua##ICgb~P zs*h`t;3)|W{EX^mnEH>WkwIZ;M|>{xgfEZ8Gd%AOXLJhnPe0qm!X78U5EtI49#2h7 zdzIB^c0(gjQ^ya)&`JhlGk|XQ4->LPh`@}R6`tbf+hnF8p@+|p;&jR-8GsSngsedL z#)xOmWUVxnEjAj9fZ?R96olVcSOvh~JUU+c9x84=Z?BJd$G*s=9ME*RQ(A%zxUKX+ z`84@*GjjLpHbU>GZS>1q8O_+o=Ha>s6=>!2YbZ`yHKaEJy}B$SOxf*)UCWLWXs6+1 z4Q==q0W&O4pcCt`ussA7Y~1rxjF2{1tDrJSNJc7MpNEux#=WMba9u9)fP>Lx8MIrJRf#sN5R;7ceG{S z%QWuBVa92&;arA+QYL~+e183AfB+g$A*UFZ8fdn?UiYAqvlgHzf%3hOJ!|JFu(|3RFONL~Q z@f5X*sTYuDuS`={3$%s!C-P(z2_G7W?gEZ?(^~IO3(l)1vkXjazFc-ZLZI4BCt-n1 zD4qvnd2-TUNHsy4385112WgrS!u#<+AEQ-6N|kPSaNTMm4KC?2Iw$j}q<{p4GR-Zd z8|GmLw5cHfr@9(plba| z<2YfrQKC$%W5?Y|$thy^wFRAtK=&g#WR*+wyP}yhUG`Uj-Sr4*VCn$^cwYTa6>1{N zTUlwG4hPg&dUTaISVk+xBW1%mUojMUAfT}1U^fPJJf}-;<4a8u3yzbvhLeiUGBr;m zwhB2-9R=mW_S?<(m2*A2E*+Fd6T@K)-eH2i1mwv?`(q_>Ol?*0UT;0V z9Ny$^7VxS=_?gea&e!CIjf0!Ypu^V$$*SE`)gL~Y-cuh=wlO@`+j2DbAcDQvEN|mX zLWUCgf@@$wyw9@sqy{(@uBvoJ~ z2jibgqeX8pgX&`nah<6TtjNt{6g%Vw~l*)k@$mg7!z7T2vu7xG^=0z?5EjJnKIMGU%@{s*(>cQQ9(*5C8{_}FF{2!5z}JCoV-3+sIV6hmfR&8gi615+Ih1cE>JKQPv(z8_E6Iiwf>{i#{@0=T`3`*l^T>Z zgsPXI|2YYbDwR+~9UBT|5(}7x&Kh!(`?1_?&c?)Hw!-q3Y!4gh_DovX6~M;$NatDB zzlSPSQZVn`KHQ#fkzcf5!-6O*x~Glyk?bJTc&)oET;4NGTK-6|**@JG`MKa<%iZt^ z`px{X)eJ{ctkP)11VWi@NtMxzOp$Unb~A2Q;LNt&!|dBwa=q_Ez9R=vyxttGWH@TB zuX1`TQyL-OQ(^JHx0cPa?NAf49;* zhViVr6$G71=1B*N&v?+udEiKXV* z=|RLPAaPvIvh7ZE?Jk?fuingw`0S+IZHPAn6x_DLy-CoW|wxxj;@ zyX{d-3z2aDC0XmKh*t1^C~?|b1{HdHylJw~e0#`ojeV>O*qt-LC?&ZR?I@Zlot2v> zb3<6m>)0{ii1(DUg1n-Eq^7i%MtuF-*TE2?!gE2*+%I&a=GfWrs<(R|I)v2x&e$r| zg%hveYVaAu#MFNDLN+r}j*bPR!k6KfoRh4Czm|!nToB8e0n7 z7Ca4g#+IURgp>}bY=1N%2Ye#P?GNX*kVt+oj)a8f;hMU@9KlF684ln&0jaSA#6el$$UNcnV^MWI zYhY~}>ygm}vS<({B2nO3N~`}z6^%uK>%3R`?3i8^myo{md=GT1VxzP|y%q|qr_K6x zW>vHq1c%#SQrRq9UkQ|h`Z^G#=rS(Y8}PpjWb@RncKUTVZg^9$;}EHi#Eea&2CVhc zFZc)$R3HZ-ze_n@taBVgvznWx@f4$hp4aI^SFTFfBuRekG@&);PL?(PW_=g2E~mEH zZVq@m)R-t07k^yvrV~j;##VCY4f=NAAZwY-XPTF$G{IGENJF7$!A^k`+(4JiRt`u` zij@|g7cA=G$<$qC)AL$*@31N7@4?Vnaekel?^|CPtWdLK&73wa#(o+(NSuttS37}m z-r2F<%yt^swW+ZheH5q*XI+Dn+Cp7HI+ zw!e%PLl~rx$f;jATgd0{Z-=d!fEwX^QxE|twhzq0#K!Rz#aou`x9SG_)QpAv%RZ}P zM>{#Tlnrrj=LlJ$%3C&q;Eitc>p}&} zir~&<-?2ffOjh$faw7UViN+Og#4-Q~rP=fY>sYkV&9L-6^E-av-qUR(*0-hliZ)a~ zw={9c!*N&@h>ZC5^S2yD8U=jOR!5XmYT+ZvR#Il>8=jYsCT|aTW_~2*>?7m^#Oi~r zjH1b4VyZ_d4(_RM)VZ4kF4VN+1N``g0AY7Kc~t002Egsq3aSwrsv{7gb&!-rSJoDM=H3_$q_X;OyC{cl6p; z#j-0D=e-BdZp)`uxXo%RZ)c?y;<7#H)c9KMPL`Yxb7v2wDFLwTGN3^#w>sl>=OT^v zNE=qTdbgP}{{}aOuam4o16t|obq`Qb)EOj|bK^{&OQ(FUHyu&5drXuvF}{PN$Lw_# zLa93+Ld-ku3##ov%$*33gd+*{v>6q|duT@9jybn!tnH50vYjGvZShv=U5~NYZsj`$ zd{fsM%0sKmJjVW73PzxYf6Wxs)lW*9kokOB4l}QYN6FGEITy9ZKJ&x2 zB2Tn{&&NFSK*(F>)3@852r?z&E-$6bbd=FcG@6x>O2(i(O76kq+LcG^{bwVD3txJ# z%r)}`^bP(Wb#E0G=el(Z1_B8XG-z;l3GQy82vE4Y26uN48r$3AP%kL0d*6ha8k-f*bK@X*~J@@c8lVp{B+4)Z7d!}rwiiH<| zA9x>4fQtyPa0AM+r*yM9*?Na{=i2b8w98mWi*`lUl_|ANwb~)ZlTQCVjUr{ud)hSF z(t`)UNv+sJsA~E|@T#_KcNIUZKYet@;j~m|a!`!G-yL+roF}tUykd8`C+|;+gkbKe zsqc%ZrPQ}M9UfYD%BDJJ^3>`?dyV(TOKbav4g*~6e(*BN%lFFY-LX+pdx!lsa?2n+ ztmE{o{oKj8`rbpgJb%&D-o7_seK1kHyi96C{+Xd^+lw-7_@KC` zV;J#xDum+d#BsW|q(xW({|bQlFW#oSNQ*!jzBtGA-s}cbEFiMEK zRoGMNh7)119g3rIFdwL zse+9_<=yM~AN?$b#*cKGIh(J&7EinAZp3knDBjt*U%Na?VR)R@?#aYgLRBghu;shWECTINN?k zR}&mbe-1g=I-EMycIryTgzI@du6%7On`MNpL6WyuGx_q!eG*#V&=E%|GfGy)^n1@| zjAvLoft#hV2ZV+o?%9>T?aJuv_q|0ZG{8Z9u%_n46JA+Mn=|{_Hs2SaA;5Tm`2@$# z*(a9_dhFxLKo$2J%{+<7Y3jSRBH5a)4No)`3D;|85Qj@9AFau7dy7ND`*W&BP(4CC zYCtD`FTyx({@bjSsde>`0&t~Pi*iP=$BMGot9MjZ+xZvFbUC(BI8u>B=fdjpCZR1l z0l6RMj)#PG!8X?rzf%g+GxF-Hw)@cuH$B0!?zT4=8bCKDRmJJHVh569Nz^6t<@uW7 zBWWiB7YDE7^40nsr10QG{3IMKxav?}iZ>G8WQnR`U(4JJY-4gEy}xc~&}V*F;-eFs zg4I5!{>F222;o_FInfWw&nUJz*@fH>o~ARQL=0Ou-A&V^^+XCf1 zp%{j(v9%6fFHwQ({M`s^0Q0oJdVrblK?*Z`siF=;P_r%Tu%-!D6cup0Rkf-`Yzx zM#HQ_^$yM$7OAVXX6-vY%#ff>hUfNk$)Yhb`bfRAYNMY95Ol5RR73k5#Jx6=t`7!Z z)L{b%h^xJ|YK^`+u>w$~|MdW{125FJRn#^jCj6$T6k5i1#n!kQLoj{{gu(+?>ln@r zR;SXaRevnl5I$x${0=^&nIP0?Gifj=9Ej(2-?IG#FLOEy!0huC+A1e>2sMO# z4i0Dj6-20q0j}SM2%?ZM7kZtG`EUeYYu0P$99kl|e|>AJwsMin__6jEVnB)=)4$;GQWfXiS!4Jck)B+$!CJ<)?roB1b+2&R z>r>1lm*viFMcQ7g=UQ@LBYW2rH5bo!kyT&xO0lPT>q&u;!-h{%d zhIQRJdNBI>>6OIS}~{|BU4|jo8VbV$t@9Nv|X>;6ti3MyGXRaM_`X} z%%h&rXbarTiVW2f_d*G98KmW!JjrR%yPw^DY&Mmqk=PF2nZrbzNf-v02^-l%lE`T~YVE8N+ImJXw8CmM zzji`xZ4PU=a} zjb4IWjHmX2hDDeJ&95nlR9RH~7Fc006V*XU42I2yJ?_*Hy$R%&n01Tbz_66n3h8V#7}QG%q8I#Ker)=!sd2e=sF4dS}ra`A}e41eu| zyzy*LzTk+atb|ybqUO)1G(>YY1=U*+;hzFMa&yl%zn7qlGAuo_VwlZ0CM5_J$^9~H zj=E3bH7qBRz2`-+f}!^NUDLLa8T=-wLL1ZN989~mN_bDEJ1LBaXjqml^|=A=1382d~GYMN=2EuJ-Qyg*ZKj0FK`A1l8t; zKl*x*<852J=1-{8U#nA|RjIopZibz&+g^0)&EQ8;OfP(2akA-pKBus!)GU`v8RGI? z(iIr_?}mCm2R;;`f_8m5jOX$2eP{qF9sQEMyFWjLMZwSiP@v~r= z?rP2)J&-xF!RnWlAehfFlq{H+AnfaWm`(5h*}m)KU*`^s1)8y#R=LaT^j-HrOEfQ7 zAOp3WW5z?{om`5(BNDuA;qpET#idWBl;LqYnQN|Wwx@c8N`IFI+{eo3n{T4sw`N}~l-sh5X~LV$WzJr?e>iDI zC8`Ca{iUzb$M9ipjUPi+m0OQ zQlePH98zsSB-rmO%71v^cJz~kXuQ(5;eF=Rl4Yb7W_<^7HGUJqDEe!I@hBWlYftK? z9y3^DTz{#rwz+T)PVWyV$~@g^lOEL!hUb!)d3v~+m$?*ZwR@tyd0nVzRY#{fUxqCiRhxJsm#^LJSVq(VWoU2dra(`YD1L^dBe) z=!mWsi}fEU^jzX!Y&APM<|H-J`EP7*ZZ3Z4?v3rU#+T`u-W_^gV(bsEA*sz?DIi&^ z8h&`K@EiF%UsPUSqu)v)uiT}kpaxk3EqE|!bcQMerD zxhHN~7wj*UBJ#~oL~~CLRSFP6Nf-li>6F>d@3!#=6y*+uYW^bdVblzxll#ZXsSJg~ zhZ!TMf{tR5lea(-9BEr2v0UBkIC#X|XzNt5X!4tImSP(+!QP=wNJ+5s6>UO9Sg}dP zqGc+tFX~%3<^=Ce`Any~pa-3orWqcuWmfO!gC1J>`foboeVs4z%C@4N_+njhDW5Br z8VgQ?eX4Lvx4FPdL45He4K&fktKP~a*1bWf*poUWxh&BrIx_^&NgrSv>6(JN9cG$5-*Y?4<`O*Usa=yLsklj(Bjq>JO zx}5AKm%VCxQ^kBe@8~L*B*qmpRLs-lh>YNq@ii#fbcXo-#JzT7O$hdz#a(Ud-E~emY=A84BIkRfO2HQ z*D}733a&@Sl{e7pJ`R>xP5hwYUWII2qrP2fYSPmG~8lYYf7-W0I86gW!eUj8$Yzvw=>b2Kl5 zAe@2c>o2>W*dL168Tu7;=_^_LlLhZovIT15`|HFbCM5|r!FM?6uzXo_|xRvz%W?EVU+z9m6Ut2~r=W&y!EGAR*v)gp}V=G-_b+Z?d% z(Dd-+={)&>%PC!QS~t8|(Cq?ql$lFwwgJuK{uxtOD^xe1J!@g7*BBYR2ZyDYLvH>yfh)CLW=2cgrFu>f5m z+Ci>DkiEp{Ce>%9**t(w!8xmL3U(1rZ}0+#^g3mfwT`5kiZN|I`_<0OG)xSNb@&U@ zR1Ssj_SvP8Vrh~ae&Z}h%83u7g{Bj!IjMj#^lB1reVEq(vMw z`_w&F+E>#(Q-2b;Icb8SDd3KZb_F8&4O97{wYd%vLZY#z8PjUh;mh za#aG2hDXbFSJKqVz^$YA@Jb+9cJ`ECeDV{bx$ZHUOC>x3=Gi6k!40)TBAVQ1}Q!lQ><(#0M zivz{>8EqWP|9hs2mSl}VC)R*)NQ`rqM;s<0=`S*0w20Z5q1p7_$&v`K3@6$&Ua5L$ zg#9beZCd5qzQ?GQkoS!i%%?V9jl)ad2#@)i)pQ0=zVy=mSX$sFhI4f#=OZw}Nr#!^ zv4!;~TKt0UCKTU5wyf-yo)J>#RgRxgFm*VxpD}V+KU#=Wb5Dvuy>&>BQctA)&ILpL zKJ0y6*~ewUYS-u?Mu*4I%&*sf*3Pa+>PqOgUYRJ<%&%ZsHjhKb6CL}rn39C?N(rvM zj3VA}sf#OHC!b#XUGe_qUmNs9e2QhNRfQ+~?$kGe&Cws-eNNIB%(3aokkq&j953?) zid8yz+EUH)-b|$&LP@%%9sht(LweN`xV7dF*B-R-*sVGqh(!-E~z@4PD0G zoZIH#PPiTJJ2t-ed_HPXa6fcP*2ogS;E)XCzHq+aJHR*Hw!P*vZK_X% zD4`Qx*#_~Ih-x1bp)K)6N{1N!x^uo{tY(tjYbAw836QXXw!w-ytq7Os4^Km>c`P&HWyg*23b>Xk%rlP zM94UL)Sr|@P@Z$zM^ZUWS$An66{~wOf)nwA&l^JK;y=vcgIGMz2e3U}CL}p{EjQXoH$aOB-&jHe5SCbYthYVq^F#B+cpTy&36@ zk8haF=G2%%y3;Zadk1{V{BW|z%k9V(+ZSTqWpKvHtI_8S2abOUoQSyfoj%JNm&Zty zHFof3A!3FZ?}@+>Xl11~uuss!M-T0JJ!wDw^j5f3@l6eh9>Ez8yo6UDJCT<4U6vr; zj;zBpTIiKtB^_8Yut1TU_I#S``N~B^ECFqdsgr<|XDyUAYYUwanxjm(KiGe!yTJnL z2$tOQHAb()rD_ApM!z1|TI7T)#D9d6Lp-0#SSStJzg0xq(MGX-_=dThV4{ZM+#OBm zMgIG+I?cDYR66=M8N%i3UrXO7X-60AbTX-tP<;CKje%P>q^0=2WzaWaZd()DtGSxo zu>j{CYwTt02idp9bIs%VixScIVpbQM+4+4ee^ zpZ#YV(E0Xg&z*pRij3`G5`XweG;P#Zt~bA*4nF^p`v}OP&kq>eC(gF&v~>g+RD3=& zm>t2lM#QMPXAT>zGQ#kWx?pmLWK{RHQJ0Li2FzZKyj!Nlp>@)yISt3Z|n6 z1@wt%FLGjVaDLTjzE@KtcykF(^7Gdb?bG0Nw;FbGt?{n_IRw=idJh{V`aS27q&v5V zn1_pJxBclY0p=Ku)#mW8+{;`9YW(Tb^sK=_f$u$2I*sXkZY)|MLiGN`3kXIkmWc-opw}w+iCeXjI%S9n$ZIH{aC1uQ};;7iop~% zy1M)u5PcQGip$GX5Bm;%4Fp9`Gyt;)#7aQPQs@tB<-dOD&h%xV&3m_oKdy=bbY!ehjcg-7Qf-5pA@oCXSz!q zj=i50Lt=eRTJW)IFMC5{6VorT56+MJGco4d3Y0RV&D#L|mtGu>#H!cRZ-+rgTMNm|9bB3GGbz@HdSHuD3X7@#QGK0t1{BR z|C#oQcrpR(ONb?|G+wjh=zOc+FqY=pqm{wBJfuvk8ytp3F9e8a3L9-2^0jo-OQZta zomIe27D`3-1=1}L+TO;FG2wfJtr1a7NMi1V_bwUzT5)QOTtS#pjn+c31|LllFm6mS zLNTP`Uq)Jy9Dki-@<3*blB#ryzQz?%yA+QiFP&GY)LT8sCOGe&bT~4v(^{>zW|+^` z5^=42jOrF}+aJL=?ahB3z1Pn6dW$l(U*}za(@la*pB3o|kZ3lGM#pK^9L$&Q!l5pE zLSBm!@I*IpKGnG+W$^mMTodt#bmDjYp8q*@$vcQrPwSkWeebb=aNdQHJqmxKE&u%r zkCU!IsdzX*)pPI+#qRm`>1-|n{1a?#DXPQu-82h@W%6eKrV&E41jJeuvp%EDP-tT| zi&;}N>x)Q>jfO>wF1wjt@ubEO@ZAu?f1ez{wDBtzCRQ*M;THK;XouCnQ{(dfkHRK3 zWCI@hkgEANI@-S`*LtXs(Lf+5laKqD*EabzI~AgG`GfEcw~7{Y!zw;{sdqpy+xE>D z1Gqi6d+m{UY`=Pq-r1%gqfa?4Q+!8^%ef=jkYDXGRbdf z7(m4*!da;smkeg=20LRyPel6m2i9uWQ0v1Rce?3e24w-bv19Wg$5kJncdjo>T&&yQ zLmf&BV=|gzN}Y|GXLFLy&Gw}irq$!7GRK!C3A6H;QtoL5Oc{yyfHza77+xrwSsvSS zV7lr*u%jJu<>%*j2{3;rBKCbgEKL>26=8tuOop?CvwS4xs&qLU8?4q=lXV;1D{|HW zGYfpLkL&#R-u<@z8*de|^@IVDCKtPIgNBnRR8YO!#a3?YK=>rkzhUOu^=w07&HHJ( z{fcOEx!F#%BG}*$b}kVM)ba2x092D>=L!+S=uc^9*X|0U+@V2}T?wO=!}QimxXyHx zZPpYL|1(EZgv@XFuiZxH+p7bk|F#Fg5&6ykT9Ubb|GzdSHowQa^F#H6=^TQmHunaN zXX7oodeax-Fm$S@8Uj`%Tn>ek?T;uy2pE#z0HehB;y6K-**wIt9er%NKODh>^VC1vxi zx=niyfD+X08a|H<3I4w-tVqt%n;D)K=r6wd;3|R_dV&Cg( zg#lUtWvej4@+#$`D0}UBMh02 z1*f--z#`9vagm{tsUVuB8Fc`gDI;9>6OC5RLF*6kNwcnf8m-P)PeO@@cQjO}r?=fB z|Mmi)fidV0l>&&RQSpXd2LS>;_YrRurP?->9g;RX2w~=Hc=$b@=`}JC7lp<$7dn6D zD)$srix7vQhLW6qcB0s#BMV#prhxdbRh9_@$W|>e9v5d&Z|vN+OBAWD%9*dO8`Y`w z0qzqG9CQ373Km!c0i$|=XL3j`i*KSXr!aaRDVvT)v9P@64zPOu-k#awbeNbb9%6BQ z#J%_9S09_f?Z>=LcI)NlK!@WSXCEHdEgS~z&e(c?>0SLy5n-oHhGOCmEGF6cQgI5q zmD-WyKF@a{%r==VK7v2m&4%;P=jm-kLaOgPbse84@&*GfsQ->Q`lGXmH3qEUmKGL@ zynn9?Xr6@SA$ri(lEd(kp;F0eAt2Z<2cOZ0?8VXkUeyl6=JdX#sg75I6GMZxJ;e)O zt3x5;3nisEZ^sc9_KPPL-n$*l+$E+7>Q0k-wYGkXg-sG$h1l}$-!Vb^E5AI+C>)QX z&k2y^Io5?*w|#X(_mgFHNCIz?M&|Qe7U)+Vr$|je6r$ssljR!GD(_z<3l+MJmg=J0 zN%ZP04ag9Jc$D+kgW>m1m(&L=ZMCu&)r*gNR$H70z8as#FIcCIWSy*#)WbCsb@Hfy=V>a{fCa{WSBz*6hx z_e)-GdcA!-LkD$&-UUS?2JG@$M%?0mPb612n3L6xA%wgT3_))`;~AxPF)8PsCTRlH zSwRZvg$=3{>1zEao>HB>U=3JKS9GcfP7cGYUi^0m_B+F044=YB%e7j1f8>h&RDgu- z?<=OUK-Uz#r>u>%Z1Wzu){ke@<^jd96fEv0#RyJ~2|g9?yhS1uvAOY)97&>6UHySh z_P}ELQ^Yo>$oQzraIepg2(7#TBzoAQSGz9tb{VCq+D}mvUaU%SroYDPhXlVKc)9OKO5u_{z|6kn+p{xSF52x z))ZaV7ay(Ac#yZ7z-C}7#JEpD4JF`Z9EijYS!Z~= zwn1tDPjU0_;M5PB()M5rep?ph1A|t5AY}wmkpa%t(TpZdQTbc7MoSu`r+d7HR?TR_ zEuWsO=X*ok?UAHt@2qR80@*kTtHafXcMR(7KUzJS6K%A*DkYk!q+$bqYhImd+}n?1 z#{*2Ed+T{PvN7#8ON4rPLvKJ7Rayc>P?p)U1dAofp8*X&$0Q+qBeZlJCBrelEx5&F zsn(5^-D(+J(Bg7RNWkwNf4)0brFFMYKn2hROpWgm;MeJTpZz=BNA-hQtT&fxyCx*=cL06+2-U%>=NrSj zjH$=gp+5R>H$g>&lz$o=4G#0|5e^6(Zk81EHt2e*;Bew?Y$9;)=zfO4W(N0=$qi3Cc@70BEc zJBZGw*J2~5fL}o+4r}3yv1MQIisct$0gl-(O}@!K*TYY4^OHg8ZKHB>6-zbrz~27v zc@FjWZu0Mp?DqkD|EENjip|wbRb4FZ+<&ENe%PdXQ+<3zMdj%~b&IKwR~nvz&fkrV zo^1>iJ;EZO#&czU=kFERh9niev;689tNqYHEBsiR#%cYn>hwcWNFar3&Bj2&x!GwZ zzcYxe!*IFzj9T?X>!AIL=X0IW2w?nw#H!WxM;Q$$Ud?1lFa9|}l#dXqi8bE)(c^^? zzUhlTu-=*R8xtH4{m+`Fg-PKKg9c8?uk$LGF~aY) ze@B-8bv)ajVSqjRUe}OY^xvagmEN7gTC{(Eb?CTxkHq7A1m|_zh2r{rPsbF35B|rT znD`tnl(+MTA(JH5UytkuM@f8gONvu$LQoDGDQd9i%&rSTF-zJUs zrOqM6C^Dk_&%giEEnY$A2F5=8b1MHI{~_*$4}DMD8v_ddr$PVsB>dM?0oV5Uk1PE1 zhyRbS-5>QEU-x65*s9bY#Dq2}&@Hay3y|ewxIR{VleSuYakm_J|_NL3_4qV#z zaO|YX0y#M#tIREu%aTy*tWuQt8B0;5RH8Vs(#BD+e&}dz&j{SHT>vIi&`0*mWI@po zHdDI*)fK`RxlQL|Jf{!TIj_s?zC1eR=TF8%%Gm}3(vuWBk4Fovr8;%UD}P*s)`@^wy6RMn7?yKsB_F0XT27?TKDt-;4=VxfkNB^fEC`Bm0Z8LyH zs%?!Gdl;x{QIW1(EU>wsMA0w(C93^1unRC71L2ZVvE&7!Z*Qgmy(A_gy`-_*?Qf`& z4E_LXt*&cOcuJR$@VUPDyx;4Gz3U3is>^x%kN-3s#5nvuH3TA5L4!mCND-_b|Mkj3 zaAKIy%zV`{*W@x{nbAdq;UA{zOoU_e88-eyS!_u&x7{qeS@M$lxkU<_#8!)cntue=A? zEZ5sU^DWkSG->0?VTC183)?i9i*h^c;WeI-=Ezn83)HibO27tM1n6JfY^90 zHtTn+mKOro&UGf>RS+hGJvsI910;jVvz2#qjFI$z0|VI5k*REdj&y@tlcL3b zH;3bxXCkXht`NYAi@n@$yg~q_sE8oqiv$DNn#&r|C%Qxi;Nne+iS+7s{B!A(_MY_p zmLf-#YOhzeTrl#r?!uJvRJ1F*=FT6=aB~m?x^C61SYL)x=r|j!mts7}(*Hn%#Tt=| z<_;O|Nb?lB+PF_cD3_N~dum~oQMzr`uW99L*^nMgs%i07hh7VQ5Ch!fvej9imF}?V zY`JNbJBxiO$n^AfJPd3ULjYtx&I+KY-pk8f!zGc#fe_T6G&@qL92y&l`=)$e?l13nO{W1RR(;b^Gd?t}m zl@&`a@&{}skzUziJYM(pHRYzI%2mQ~h0mkHO{kvl2%NV|#+GcRiilA1Q>LsKLsS1) z>>SP%+2d#POV84~s{(=wD47(lnAV#E5uFC?biXn(RCY%Vk9odwAV%PPC#cfTpDU;dy`CQAT{`;IkN@+pQC z1kZk3R#l?7%lQ2nXnK?f09ljKWWiHa<$A0`x~!&(*GCJH$}(x3c{ZK6pg771)x%T{ z3t7QOzSxhYgc*JXkbdHPbuJ&Mju#ytL%|dv?)ELV3So3SNoTTJXkMj=Dp#*l#HzBi zo@yBun)OPn z6kyLlm1WLuc9#z*7fc=ZC&<(#1; zMupBtSS4ItMp5(1!?vrVvUqGT%RnTaDnZ@Y(&@wxq7Zp77?n=ted(og*^JV5YcB#u zS+>?OuXBNP665$$icg8>W}Qf(!gi_S#;v>xZbp8##|<2F((uN~Emsx-Cf!AMD86HG zf+j%1Sm)cYizs=N-~e7p<>CYbthZD)#tVXK_LxIPX!l5h{O0`-mgk|x!2srNa^rcl=y9TwYB#^Z;0Uh|nCcrJ0OqPWU}}>@Us$Yk4gke6Z5ukT&*| ztU%d#Iv<27>+)qraEL}3qNJ3LX_3)(mCNjjx8XD2E{$qR) z1alMu*VX$v!vO!XPGKNSVQ5Fj&GE_*;8Ib!w7Qju z>hn1FiaABdcSR@-?#*zIhVL8*echH;hJg7P~>y zF>CD-k%dV<^JR`){QY{*^E>Q=(=rK+M2=aWd zBS*!P3nm?pUV(Uj#8E}>R|yu57ialokCcn)3=akuVA8JMMYQy&(6-TY2dfVRAvV$4Wq2%N-$)@z;OUha$ZvFj>DqD!-m$BW9& zBlUfI>t*1Q*u{nqM6UtvyK7gL&uyk{#J`05^NpN!G9!a_Y$({m>}B`hl=lP5A3oCJ zC2CMb5vUq7=uCF@)#z4b9-Wv4+tY=j9UU*aC@py3KEFyGW4x_&*rh24q_EyF1cL4n z03d7NGXHDMOSDY&Nfet&D;q@+M+k*TAouoc11(-3wHZ}pcO#R(ZTkG#>!H~4r-4#m z5rXEcPkPR<;?M9{*wIXoFj1UmJ8f_2DLAPH=C30S)@kK37JkV+B8xtg-gVHdfLs(c{vhgb0$?w_XVK?xHLP%2;&8@G!~PJ;?pT{bYjSi z(xPMI#B^wv60$S#C<<`lHY_2_PCx{Kf_E z$yMl=oQDOOXMBns`e+|+PUM{|kfU1ON@u=)dp+^&)kb4nsuZ7yNe22-=MxR#Fd0^R zBy`6*K;^QgSNWJSAkSkt*aoXmq$A3XG$be&<-?^`63iuQiw&n5RQ(OoH!E;=zCfQJ zMJTX!T@6kac67T!9?Rl3^?dP0^%6X?*p@ie(Ift2UE*#R0Vv}S4p(N_E}gv0sZ+9r z4j-;PXd7n$OIKIDV4q@^d1I{hR}={iPocZM0N&P<=VxLo%pCX%=^gVllTwh9XfM`eLx8N0VsToL#%tM(mv9iPXJ zpM4N-=CFVA2UI|TxHDt^4CV5TWE56HhU^(e!co&_u4+M|*-+`Kk${&Qd1?$_r~?X< zMrD{$Ac5V%Vpo;Vc`OiZVfJeF>GE7Vs!1Qqu`{K>D^2BcxBg3cU|Yp9iu1{2P~#&& zT(yv4rHcDAWBuW|6vVzl)4DGcE@})MP%<>R=qOj^K1Wgeu_KndplTw7CB1&QE}jhmI?`C7Ng>*#>#PhFR#y190v_pTMaKgplCG9Ihn5tP|`-8p+W ztXSRiKwy2jE^wBXqf1R!K4}a|iL)!GOk+W$>%l*3vWC%2+OG#Pjo@`gn_rLI5S|gn zfOS~@TbNVzW%N+=pqsYDUh(h}i#RKyn$ajC1_GdYf%&;sVJ&-iM9%K4UYIAa;)$zpd)R2eu6BXOcMFtXk93k)1}c( zF>r*4*IsQPmv(?z80ey8nZ&fgq3_7SQ@%5tTHx&r|M?vOuaiMp$)zLE9Mhg*Z%9S{ zTaXWvVUvWo3=P4UND6F$Cc;T|o7+X^-e#g|z zC5F$`EdKc$F_d8kvYE+tn13E?zP9 z0^sH)3xN6N6lEi=K;N9r&`#vRyzQ?hkd!oq&Dte~&qZ9H~%Q zU^o%d;o}u}fTR$C8iV(DbBA(NnWAtJZn;A&rL+WtMk7nC$+3pjt z9Z|{_`cZ7FdWX+rR96`95y|qxYtk7=I`P) z*1Rt;2iF#J66;Kx22QGvD8qp_`3O}fj^P{nXPIJ66@CUoVAa0Y13B{fHdD9t`StjQ z_{9u7PUX#f>yMOvlOa{LMRFE(MsARY-ituPJr_#+cF zA>egUh4f4THU$Llci&=h+i&I09`lr1Ej3!^0#$F?GTLP(uuu;KsiuiIH7Bz)!JI#7 z^^PGw?x)f}qCvqz5k@pq5+D}|($oB{s~AIAc>1(#HPBH8d3-}WnY~GqS5%|sT!ZCI zs>I{(yaTbNCL1EyT8oxR{1^C|-1RrRfN#T>AK7CG3T97*6_ARRhO#*`s%SI0(aKv$ zB3;pBgUwW`Gn_c##5B; zw|gYYJGbDomO9kQ5-(n3uX?XB9nesQ*+QLB?GOS&*;QIovHUvNlR&`nTNpNzFi;Fv z0=1FCT7fD735YG~ot9Fmnmi$t_skE&{V<Y5c`POYfbw1E42fc?~I+2firNdz`ieHQH-uCkkn=agZE#lteE@e3K7MI-ZzQ z2(w>CVJdHa_qDCi+w#K&r8M%f+Wd8>{Z|ZM49m}eVrV=j;DI_3feJOiiS!qp)Y0$b z>kW^d$)7RO0$YgT)MaG?s*>%kT#kGcs{6a6nd02FyVND zr(F?A=y6&se*;)a0RbzfWL4|Pg3UJ;6ZJLhEj^!XT(6g%GspY`x|_^bBPit_L}{kl zFSeC`qK1@7-Z19+PL3PV!f2uTipPJtB=fxycEtp9X-b~Pgl7L<7v7?M*4?_bb$}aU zQGMTi%2N@D0I(bq*rqwOSes!TDO^=~6@~N#)slbhg1&_`$oO8OePA7)IWtcCrz4d# zKnM^x{4&y37j;7*D^~%}dj27@f?`4-NKsB{PTV$~o-pCmb+F&qN5UC_!eJT%(PJ|! zooC(*GX!@VvDq`_b*nCls@+Dlbn&v=`S_XPpiZ+WhIQKc)>&cPQRb)etX?N>BP!r5 z?-e4hWyt9TD@oxFz;7{uztJj{_o@t`$4mE8>uAwr{^K*o3yGe|FgkprXU7UH-(b>{hvLK zklLTxTzNnFA6nFZj}rc~VEq64$FEr_{08$MZ(0TG*KD^bpDXs2TIUhQQ(#Ot+#xm& zuvC><=N808A%v0FTi15K-})kEO2lw3{f!t6U`Qkz3`SuxKX z+OTmxgham|{G7`60jTwTGx#2XMgbuULWP!(>CTgkrl#H_G*ciMO*pwp6;0Ok(cX~+H3+^x6fGc@Qc4zp5efi_$XYY< zPW?VBxc3%*e_~Ve@^T@qBrv?UsR)As0S(}|AOkek;NRXIxV9;od|vN^KsWZ`?}76I z6vxP)(+RY08E`N={p&3IL<~h|o>SccHymTc)k^2ejGS;_1b&?uQBV! zlOqPSH1RCPw5?tbl>C72N#rict3Y2~XaG9N=8E0`w<3P{3D}6ZU;fT%)7A+;{xVsZ z*KD@|_%$`@gkQt}gUgiG9q{&sM!#1K=sK{U`!Xb;75_ufWH=edm_1(Z4|(jpajXcy z`Tgvhg`wO|{xxorRYhg3)rplBTNU!xKIBRRLrys@-9wv#@PDrj;sexAJqX3w55l>= zVLVAqw-pI5q7y`i)(h zzOIGV$@UZMk7e;=!%pMbHEffU#dddO3mNmni&y@kg@~-nkP)-5r8y65d;Ysh2EoEdaA0;n5H-o~F?CN^ICAq;jp z&9-72pDgiJI~pGlXRMDZ0I&g2!7oHZrI`MI+@brT=zZC*$hJNEUNn(ccCx_i%Gt;i z`xGqC6{p6>^1%JIZIs&cYrdI$a`(mV7`5{eC)uKKgc|cgt;Qmz{H^o<#ob#+#kH*a zqQM;!Bm^f3?oMzI?jAf?2pXVqmyqD@9^BoX;L^c^yF)i_jog{F_S$FPbKiaE{r$%1 z!I(^%?wTcERekl7_*^Lyc<`+vW1@=c4ZQ5cp9Z%2(*Y-c$@TwN`##MJ;B?W2vP6pT z2d(JswaY@u^1e@o`Qhh(M-2czRYO%$@&&#M{cs^_?wfCK6!+;S#p2(zholC1K}810 zJ(F6uSn~9Hp1RFlEejuA?Co7v+IatM>+|0)9IkP*aStHvS2leAlfBk6w4X$93^^N^ z%$)yM%g+BYCTmT4mZ0Jat+yQc+iCCrW;FKaKIJ&73L1gsrxf=8xeQC~h-%0mrlyjH zshSs%(>^05jN3G5yBI}B3nh&{7E6U7lnLO^PvpW6M=hoPy8pM89@PhUxSXKj`3AqKyCK5Z#C#1K#$57! zUeWi$u=LSXK6OK~%Uo7XlXN=Ncsf<_4!sryyOlQj{4m0VU%!6QeH>V(_dNj038_GE z`n$zs^by%);@d&wvf4d4el?$cuUa=9-;3bm$}Awp@};?6E@%#A&}QYyCCH7Z^V3^` zGhH&}j|5(+jA!t6=1quglkirIP-h6ze5@DhDco7-i%}GT^jHQ9ydaMg-Jp)9me(4J zd7Bg@y%f*pT{$4&!|_0v^W+s%bce5f6^u&Td?Oz7^0VeSI}czee|!az7~B*TV;&v2kD3yjIZa-noN89V!dFG`cX=xSj$h59qOo&us0t_JH4F#{xK}!)7hCOk z$dd?(WGUn@AGcu=(}&+Y&DMqrPu6NfUnQZ+C)&){$o7X5nWjtIP>;q^hcz1_q5rfq z5^6Xgv=z2cAn+y40#49?x+_Z;28mA=f+=s$3B!59em;L~+OL4CpB){f-%*PJ2%}mfAoCMveG^J-GX)I@w zJ*x)LEfUoF(B%_Cw>K*T$mP)EJ=}_j1OI8)HB@8D!8px8PbpIl<7veNm3GbfjY{Xk z6^VrgYg%nc*F#kM=(=)bG_b{32sE2H;6Cd{p=b%}+ZBrnA&(95xC(C=q=zaeNTXnd z;zN6_D2D70c?T!WN8VV1S8FhS=>U-KX<=_eKG&;od8~Dd5qci% zakfS5OgKnlY{QfVZ$W4g;3pw_ID6e{nM4)9$D( z?|Q#hFYuuRfb`t%fT_1`!9mxvNgOhsrKU3nRwmcXHjYmxn>6ExKL#X;Uww&!?%zaA znC5qOP<1^l?Nx6o8ov8#>@pXp92M_w#1h>|+fF<4dHP#C60&K3q_6x-uliqgg(g21 zGsx@UY$b(()?kb)Zw zJ43%mh%DOxfgD8Uw!d^K7Dt`|FMqZ*X4$h)XKNX7bXm3Wb4SQ{dx-g8lLnOw`~=oZ z1I}Z50+Yd=MBaU@drC&=BhH<8 zzTC1_iSheR?`S>=WSv38G(mRQJHU&kVC;HHon4J@u+pOZ15G4MRtI-2edO6{xs3bS zR*c8xA%Hf%=11}VQUZKM09-L+(%Z@j|9jwD?al&0!L+4lr5(wdku_h&PKt9Lq~iBk zKH-%SFCigJZx0WQZ~eF;6G;26+(a`A8p;re67AN=lVMly9Q_QZrUT1@l!=S83~kBc zWn@`vR}QcDKBXS)j3gPM15N2(H9xx3DU{#o_Fc`<{#=Xy`EP|ksn*12x{Bo56^2@o zx`Mmw*wms?>xL&i!A|1~1k76Esr8?ju}D@KL5IA!gp8SsGU=y3)V1`$XjCU4hk2h$ zevFp-Kqw|;fa^f`7p^ZF!88 z{SSohikJRcdPiU!Mpo;?{S@A#?>Upz5}x~W*DvsFugY|HUYR`C5o=+bdlAW#&0NHb zxO#3emMY(ijCBc9|GDWM4)p~R)Qb1x2Cb>(4`3AaLb=kH%vIt$4)`L~S*L8@vk?V9O{NB(F_M%k1Bq(bW z^rrO`nwCJmIJB1fUUxkIqZM;+6--2K&_3Y{M`Ew00rgh)ej7$M{-Zsy=^Eika9F!& zZOf!@V0H@RMb# zM|@{&j|+=TmRlfD7R%=(u`U|%TuU2TM3gtWf5i)LH zKOopH%GdTcT888t;S_=$=r>V?c4jV;>c$%BA$4~JwZA$=kt<;-SxNyUfZNU{!xo0x=14d7 zImj@(_{vVXEG|e&MxE+9W5#EP@&gf~o%z|$ALFm{T(gAx$tbP>t{-IvziZ66LCbJM zxt=)yi@rn1{9rMfYRm5%*73NI7jd90kf9mY;C+#^DTVSyy8S-X>{n6+AO<-6^K?_; z%udK7ng45$eqvu($CqN1H$Sh0jM29#aKF9whX_$5`aTKKS!e5W8|iB|76{fRd?`n1a?kK-vR^=da5^X#96FGwrd`FV2o1E zYo>tb&LhqFtEZ0Haz%sLa)ZOuHcv`q^_{UNamxS!e%T)nf4-2Im-}G!%gwQTUe?R6CxXIHPeT*pKOGU`>ci%Fm6># zlX*)i6wTKh<2bP}G3oQySu+9~N*66Oyx?C0B|62qOph+#H7Y;!VNxom3hDxS;nX^< zMaJEWartb|ml};k;vNALEx@6b_||R+*ZuR)xN*U1(-~gxuNO4;00JSFNAbf@>KOYc zq3!mev&=+)|MtpG-7QkT`5N;u=fj1*yxWi6PkCzjau!cpL+Tcuw7#1?%; zF*wL>fXzsLXxw=$gE_-4Kb1}rm7t)$h4;pMPp)%HjiWlW{Q-6*hmzr!$fy1i;0f53 zT^P>Qt$`px0{4tKAV3+s{`^IH+;Cl#@<38&@CCC}MiaR^+vn@$g>OmcBkv!-{Ms zm9_V)2z?;A=jc|HCwB@1DwDtFuaU~Oj~#?=k)m*CkUUqf4tw^SmbKE#^y_2al3#Z7 zZ7J#|9}`S%3w_=xrt;W+CL@*_xri!AIXfADfgi5~?ZueP_7T#(+vFHZn=>JB-R#HO zzgn{m#~v0P7Ok3z#2HJasgX z=@>RzY)tcxpDA;9s*SQ7=xz5L{)$Bszi&*$-=p@qX_5XZOWu~>B zDFQN}%UUuUk^m?0(wja%d-(wclM7)O&?DouI@txlqi5HvC|l4dY}7Kovr@-77HauK zE_8cA681qG_L>2-pa%fwHX>Dvo%-s%Tc+QXpEOD-Y)l=Vl~TYJ5^dzG`vP?);+nt9 zoOU*}Hcw&SG4^~GCfsvY4F9sN;i>sj3tx~)zkYU>b4?2|DWcwvRR3O_tr-cNSd>Yy z;$wY6f+fFpe@k|~aqG0JIlnM{FU$(F3QnJ+hOaRjvKgKSOotvLwOp)#hI_$5Q-9_ur>PoQ&3Wh(J z&h6dVG}LTW8-7SHH2ve!1QHjRG~joM2JDZ}4KaZsy@M;HV_DuUhc1@{q^o6SLJ7jp z&{#$6BZ+9K0jvOzcLbF`@Qme}FOgg~1YqPTRF*cQ-ZbHuzEUFrm;;dV1mH@|Z z4I%el*SXqMQ(uWT=~b9b0nzNfoerGsp-i^PzKt5Lm3ShLMceF8t+DjJYnBx_&2S|{FABymUvAk$34FXa zO-o6!7&{v&%usOtKnb`WbLibymo@G9TorSUcHn7W`?X_2;`~?3>|yH)8!-;nB^dc0(}`C;;YmQb7dC5T3->SAF4~pr50#APvkH;MMZ6XjYu746=CHqJln?5BG)q;x2!ema~zOlO5$1= zXhf8wbM@B~zHoh=!~ZSut)(4V5cBg*Wy5l1FR!J;Z=h_uud59*{^rTF)t*ESBKE)v zhnYx_K0BI+A)kOI1h5j5Vt{1Xcib*4J;{X3Kd6Y7>$NHkC$dmufF>7AJ)pYQS+|a> zN^{rERlnGZ^;#1z4sN5HJFENCX~o=uQ?KYbBhQ(or>93)<;9^d%<~&`@7aSP_h%I& zV4H3o&)M#I8UM8;x@T<@gSBI8)CIgEq*g9xKE*O3MAjxHDY+DXcBgdZ09|=MWT~s7 zgZALPV{*(ym%ZZl08-xt!sVbRRDkA4XtEJ7*KOAW+_LDQdoWUDe+r8GWSbc@|5sfs znz?RNNcaa6t>9h@gXJM;hjnIxip6}s*^pX4!+G;@!+oofWSr>Dv<1EL)#P%jd;k&l*jp5qP}gj4mOo1PztV zI=Rh?aTJLUSNPR1z~fM<=#x|nNqjkaY;P1{WJ1z#jx!=95_o)nsdrxBLy_|bi50>1 z`FGa^?~0FQET{W72IrI3u&ejn4z4petA4>0&1)Vr0rEARzw_6dc ziT#~Y3@N(<<=TSx@)ma>~o1=?4O^E7H!B=W`tP5X-x=w&f6q?}G?`JI+f`x{H4q zR2jaOiem<<{DWCuuh*T@K?KEfXMVj#8y3J1H3{HROB3)cN2m>F+Dq3jbVuDK8I304 ztYp2>(Lme@t`cSpdie zif#s=eE9urbK|RDuBniWMw(=fY$}f`C)$Bq?8tJylXXB~t=NF19-ZHWU-HoRX*n-d zVAsj?GAKA9vgh=)`SxASzj5xd=(B-l0nC< zM^1J4LZf|_yH#~o?UhLSUB}VHJg-}GYVILR-K-eKD4mbvZxu$l_(gj{R*v1py(2-H zMsl2#;~~5O5V9qEZIno|cL?Y$HtBCDW-avRjF2&FAdG4a!(SoL~+&+0g!Q7_{?ADRNK#E^49EA1R>IG3!H` zY4qN1mZ>F;#}F>H<_nXtg~$#bm6fBpj)oUU9MIMWPz##4|b_*n>-g5vtl zgpvqKk(ehFGr0hor_l}`wLKk0*aa5_Z^g*M#%g<(;SXFqpq|_I%?_jjE_KbmhTga4 zPTn!OfH>@qfO?0x`z<6S|Ksx9Y!IgrWHJ3S+(}@|YLGYq^u0=Kmp3`h{X@NJ(s>|g z#vw{EQ*fxM?f&yRZ~u0Ixsdglkl2sV6`hvk2&Iz?jzo8DKnCu;R}~{OLXA-vla5n;8tmX#~x4C5+b-YOk_M@d?REPg8=uv?0M(rM#V^j%%7DB2e zb!M-~{YksWuOV~lFYXom2ne4F%}Zg56c};S?6R-ZK8RUPJstxtWW+cEZc0mq;IOTs z_9rD|6(=2nKXFWk4L)=A-9`?-!;f8-f1OYKw42|m@H>kiS5Nj_nf(HIGt_Tp4b!EM zr++c{M!3^=Y)f{O3Rr@gowPn(ZcuBzJg}pzoDXsAXF0q?`iV>D$Y$(#8c>?ng*QM} z%g?+z#@BJ-;vxJ;xx6SX<8OA9E zocp7wPi7(rT}G>dji1)KJ1?bj;<%a{i@z+QOu^Ld&fw6_zOe7jl=^U={(wzAbdR6G z3YaJF9#Jsg(mgAO3))+_oq2ks)q#~%q)JGn9+uTxFOWMwX3nLAM5J*gc*CwziKddN zk7PH_$WJuLdu1s3;jJ&F2;rsT@@;B*4qE85%N*kTeE;N%DHGAUkRr|3{g^H2OjU@6 z=|_;VM5JqPn+?6?1Ayy$GEJl~|s}sk*x184tKvULQJi_zp>G@#~{YCpV zMXb@~co9Rm%1-(T8({)01PiB!k8@tLKt1f1>{~7Qp~+g9A8sho+BGxoD&N5^Mh8tY z@_C7wx|qNNVLIC1bn5i4kZ!TrzaD>czDHbkzT@?oCIyT5KB(E^>d>rBW|>&xP8kLN z9Es4Opx%Xvk+tj`F3Bs|1aQQC|07yFJ&rbVNLG^uxdnet%tHQAecLo9x35fM_a)~- z#M(=l9S;g)(RmN8wfSe%8XD+DMN8JpP2AOEj4=Q?=8H7R}$T+bL=j zN+D~f5TxMeWrDe}(sx&!h3gCFeX2i3jfLUYQl~w9zvlne&!}s1Kcwgf(b8Bim~ZIW zTe)9jEQRN5mZ~%<2Tij6+R6+~2oFYBa{n-gcJGj3>BXf<;AVdDaAw$Ukp^FWP;E$| zV=hjQjW!`$aXWKoP8gH!m%ua~?$^@~K(6E2~v<;QK0*rpFF8Zs} zL0j}SDT{AA+QH^=js%$v9DCi4!CXi^|P#39KV?Z)3TnM2ZO0QUl8LZt5JVtw^E?Zs6c17VH|lhGqN zXd-2bF0IOrGV@8KQpc{W6NdH5%tzX_7^Z~&SHjuP627sp%UzhFGAFSIP462h+*?QV z_J@r)v9el|j0qklMK{FpDEaV6-#XD6P>s&38>g%lPO)pgNA0}qG23=JPxN&^{JlAA z>!K2=cgSV36W*#nmGt4hYCRa475Bs+`pD;WS0`VC%1bfILfcNdCIxB4BH1o?U6~*v zy_WAAPB55lIE!Uct5QCm)7X2b%R<6(h}lXeV8=L^YfWph+7cNPQCTm{yXF9ll$&& z3qO;KG<#I!&rI`;bL^DDXY-aw*5h@bL0O>TLzj`nF^jIl;3Pv1Aj7qiN+&QaPHKCq z$+o5etWe{22J+>R@SJ@NnkT(}RrdJ(9yan}4_ru|5INY^HcxHw>2JYysXng>k*@c> z`DG$osf8=z_0(QX*PZR?504a)K#ZJ5)5+U6D%jXcRcl0xxo7IltglYj=kQRK-D>aX z*)^PWLjWoM#MF#U>VWC?25XkZ`=UDMvp>hL(E}G%(boaCCl_6C^+$6c`=c7`0hdi- zER&SFCoh@uWq?GGw}Rj@c={BXs$8+;p6`{<5G%2K75p_A`3PmW*13@;^}*Y?a{SCi zGyQnAabW4X;K2^cit41zuukL;XQ3Q8OcO=-c(hVKf+^E+Hf$Zx2dSosj^^89BXvSbt)qB9o7BZ%ESMe;#8Hg^v_NAJ+m%0Vk+lj5_3EZBu zOb>RSz`xIT#MY(CF}5I`xt$#gkkTDwY43noE6s1xO#8@r zgY7)*F*Hl^H0KHfogbQCd@lb0ZrU!*8723&`PRZ`LrpR(AQ(oaA+*_|PnTtUiHL-c zuME37a`eG|IB!&vQQR%-iR^rIVEFpto8{hkr(G*!T}Pu9ydZ6MRl`p;NvX>+2j2!A ztLk8>y2e%FroO(U~I?Ez)Ck@#RJVHsKZ zE@DGdp*MjM3|-Zm)}-4kLBx88mPJZh)1L7SQ_`s(25+9u;j#Ad+uzhJ9Wg=`z%Glx za+D@)hR=fF_nX-0=7ZNN7=sW5Hlv3NxF>kkhF*+Rh2=!1}K?R7j zxh00}Uytf;C*MC3ff#U*@qQ5n#P+Gx{nVP*64sF0;Oi793;6!K*&I`o;^P zBqVwxNt_mKU7e67G`#Ko7rlzJ9d%9#AM`G68ewT6Q)f8}om>+lp> z`^w0$UwNqS)4axbYqrBgHi(6%0N+x4m?LT|6nKf3y=Rq!P{3_HJB@U7o?Ne3Wr%>B z5mgN9P*|Y2a}O?sN7`pR^pMnvL$&7VmeNXVutnl@ScmV=Z1kRGV=AbA$467|77Jvs zi3rtan|Cwa-xm~Uu<&1Vn<*yGM=Xk2t+vSSa2| zW!c?S(IO5PQuy3YTxjo8^-L|qN{FGJK`C*WC)(7x5zkW4mJj++r zPn`d%EWr8GoR*>cF-UVdA)0ju94G8e{zXZ<=DcQZuET!Y+O|5gckUn;({6&Y3RADl z&SPyH71Ee`nEHY_9;Hb%I_ZHwL)hsw8vr@c zkABdPH8ASF<$EMDK*evv90reh@KKZyq1B&;7Nn*E+en!-w;A=l7ND9ylRpBWxL=@xmm^6}PHiMdOU>$r9;Zmx zpLX2#aYP2cy-*%utnB!FHgMis6XeK>AV)=4ZaCEs@5+D;+ea68Ob&$YkCG>7W-IyGo=)CI zq2kLIiOkyewu_kdJ8L#gb1QtFDS~L=ZjYBArWY?d>+tCY+_ZuWid#U3p|G|6BVEpI zc5hk|o_pENm(y~A%w@dqOIy3?p^6PCD5Jtl?j4+8(g*GovUoHJT&yl!N7nDAO!KIo zqc(avbcR??vuozHij5J>)ISd9%iYGkXYV}QHHQbkapMwIh29%}5)Df0qk6us zYmzm%am?r6G@)ptdpf0T=xcrA&7@b6-c}lFh^oKPQFJIQmJXWst{XCNSb4Z68aUq> zpSiA@flLUIFOF2bF@9*=yF;IacEDUGhu^l?YFt(njHrmNW$cEr628KAKx=VWpJSPS zKf~*2pVa6h`f!J(kU3Xj$O#4{BP5A*;y>nMR$wg=i~l10=|r@VDblk9<3q*3_ZJ_I zE!X3*dNQv~c}H%g7Dn)0@9%67t-o1H?KVf{%kgNph0}l8YRI?s)sCB5HzwDFcrHF| zR%Y;fveybeY{e;UtN8~YZS5M(B@<5Ra#{|T{xFOu7xE;Yvv=#ZTd3-&?VVI0EF|T( zX<|})TH&0|@0qrukYyy3A>!Hkf@hHcyUuxQ(f*?(!H%)9zoZ9AxX#GiJ%03V%hbKf z)0K$REG&j)h6 z%_4q{;cUw&kA3f{V~J~+h;A-F0^?tN{!+zBy?D#4@(n%4JYpc7H=EgLPwNA?n81LCk27VRh31=11f~GGmKivBqUTsnandhRMXU(kvmc znMGuk8~1spbR?j%lf8C&Z6!zH`^N$n`*tB&W3=GQlJNsd>5SA!CBieON@<%_go&Ay zUD+-Q_&UF{kVI>J?^V_B(GK2UnDlaDZIx=jnjF}h9oTCKns`Z*^GRn! zUrb8zR@s0Bq$5fDaTgADb#~Wg2OQ7guPqWC2N$5fi))E~mKm0pd~vtTuWh)xaDWDM zlgc+6i&*wOc|W`-Q&^@3^f_t*ycXWr>p+Z|5nRRIl|@zggo?mxvsl)3ZjAp-5?oq5 zpvNjB>6r7GbpZ_tzAtYwE@xMObyV(o2EVQ6-ptAZ_o#-Vr$|xW^YA*?@T}X z0Q1_`==>DF@uA0<-5e_i%3(UI9e+8Y0wXo~J%Rt3S(Zi*ivRE4c8_F|o?m3rE8_#}1#()Q}x_UnR;K-Z{@JKxtFm zufA}Vcq_!oq4?Wn%0~uOp^%TMg_Yh&Mjdgt=O~9}(G+H_Ezkb|1#0C~X9RvIJ@%dV zA2{*h?JFtVy@xzJQ=mx|G!Lzt9HL8haC}&(moZ;YU3aBm3I7o%u>V6c$yGc{xv~Od z9rDtSh5ipTH2M{4*GG@tE87hR`wWHEW>c+0#M+>MjiRa$P>wMfQXk&z*I^X~Q!4A? zWbP?$0*k@OIijLXTBUeM$)|vQP6f!Rgk_RDKZ4ELkHCC~^`07^DXsaYFYPI>KQTYP zbGb;n+GAe`*#>usMRM@i-AeG-E>H%ezvMY&H+jBYOdHZiXXK0P-Dvh*RUD+c{Yqa> ze@-{!B**^xi`{mkw55~QFQg3Za+snP)01qbuS*_ixbWBpR1df%eom>V4aFh-&C0sx zfn?t)pevlor&pp;wiaiN#YE#FK{y?BhHkN>CH?uajqg2L@V$2V2Lf|%_OodjRp{ypHWvvIoB#)gZ5u5u{aVv^4rpty@6PhK^yG8p5ZzhP^6Z!ujt5pe ztVuo(F!?kd+ff_Si=3BU`&E!@Y5?+bUET;s#pb#jS$D;zJU1`&a4t67Kl=ak*^zT%`=b*CwBHl39J>44`icuPQ6k;H(HA z$(aBHa5v;5Pn)zkI169ogmS@lZ?FS7LkJlD zwmLF_014#Y_5)eKZR`q46G<c|tY;Lhz`sc~1oUDaNd`L0GC zQIB^Pj5JY1XcS*>Op6U%L zl^0!oz0pyYj_$Cn@ak;@sKYx%sh@fT;1ZZ!q*ZRlGu_`Y?l1cKeKCiVFIZQ` zWM7|nn9r}kT~~I?zkAhyh`?AC5PtnJ&}xpW*d5Vh*<*FEiGwo&2=-v zy~@`>89&c{s(clkI=i*L`ADXOnw1E@1`7Ja3qjfw zFI0w@ZYue2mcb&ISX)hi1TMywON||xM3=`)zD&i(2u?t-UV-_AvYZ7-I<)$k&Tm}^ z$lEGnrqB;5|3UXfmN&Z5LCUL*rBZTprc$UtxxLM0L{))dF>W8Uj$c6;{;zlwghrJt zAk3y{#1Mz?)4yK8ToC>M>f1F`>J~==?Hq$PvB3}NurF2ylD8MdQ~aDe4Pf8vcdj1# zxsqu0Uhm>5YbSrjNRlY;jS4oke3Roi{5Yy(LxkS^lf~Z!hsq_hOazUJ3a({SUgnNT zAR6uFFp@QTv8N?Q)C;fcgX|Nj;99y4LEKq+y|!1q;5S(!*HjwoS(^h#Mv2(H)m3_o z0$MmS%}841>n)k2!H6M4c0l?`4sCyX<*(!TEzBsq~4V_xD!j?8+P4=t8i>mjK9*)nukz~RI9_sc`Hw@UR<4gdU9^UVnexUfteGagXqauYX zGm?A#2iw&eBYGBO$MuK*wN?4^EBx>(2T|6alkWd~eH35&?JKRwVa@iA#+?o8PXg*j z{+Zp4F&aoeOU$89`p-xG{qnz|i8i(tpC|cTzP}YFg+Ch;0FSPvf&b6n|9PVfCL-HS znAn=&=NLb+b5C2EOabLq>!dng|Pk7;)Thh8M< zv94qLuj0tOL4<#gW`0^%maC)-xAZr__0LB4@0Td<>t~H#Iwa_j>OEq{@A?`%fr#+Y ze#__#&q@avfGpMD?^}{W#Pw1vYW>EYCihqW1O}TZJeq``H6<;kYhtPys-&dk&kuUp zZauWWX1|8^(ex8DT1diEIqV{SLIUk#rXZmGB!_buHjScodIKB+C?FOE$th@uf&M@_ z{#CE9o&$mo)^RJlP64GIpHJKOToLL9y{7Q!X;2h_QS z0G2UA$)t`iwzXJaL(odoYkGC$boGqib~NX2{olQg28<)nW>)q7$|o+_0$#B%STh#^nrQtVE9_$@J7W%fpo1o}!Dv-rYfUy~lN^Z%VSukn98G>5moDO} zRbkZnlFhaee`&UshKJFGk4mz_p#5Ve!BFfM8G4?1j>Z>@OFT|K&I1Kd>qxzZ3kgUwy~L{%^u^R=09lKH-H>}W<$r!U_;i4P_5b54rZoRp#&xLhPki}z zpjmtCZ#G1g5~WHMfruL*iijS;=8Ko>miQM4_n%-Cf(WmKme8R=Gc?2Vn~U$yi!QUgV$`ot&`o zIqp>jtzs(1+M50NuclkGqkhgNM<;)4MxS{^h-jkq4qt>P7|WfeQKTVDp!zhtKSjIwLncw|KkDo` z4kG+V#iDdu44HMyVLk%IiF%Syy1ehU?s==vbYU=L_p9Kfj{=8OajU7Tb0a07PU*!= z%|oBK;IJf_z^@Ux5n-rCJ`WM2k6w}CSvP>y+P|1t{~mT-0{&|GBSc8kw@#MDerJ*2 zE#bdCgUkTJh_W5PVq^{{+oFcyU+n@kA~0FWI8fuq%O=8veD?-|-!L?{lxE*ZXG?y^ ztueheyxrSW*2m3!_eH#?r)LXT%oy~vlO3(LY8&`T53UG#wn;VrI9cVjOXTjfZTcaY zoczCy6kH5{XECt=F>iD{%m{5bg{NkgYEnqAdJTVxJk!; z`-QTUKiGZYLc9Xu?E5-xqzmfZ9r+n~R?|kQdjS8yY+_3 zO9Y%j%<{tyZ&;pjP5_77AYRSUK>ppvi#G-oM=b8=+p3Rx+sEJW-kcr+ilAJdmn0_v zQl78PVoCUFQgpQZSMNVKsw-M$a+3qM$0I%>&O0niZ-9)(dVRn>n!=cLU;n^*`*DR- z9feW66Al5$TfE`88=KpHVM_=Jbh`U^aeFj>mS(8zDyFwk4=uXgjt7+SN0z`M+F3h= z`ngiB%FU8HVzIQp+IM^Jmj9*z)RZG(yo`Yl#*iXm9aeCQ{ihpJ4&lip3KAKdP~VoO z*O9z>@kO0PMwwa`#!w6T_b711_$%uv_xwCMSw}2G-}$FLq3iaH$fnDUITjMqbjxKL zmGn3%%iL|3KZ07B4tol>Q$ZX5fs|8)?3AZDUciKsQr(Wny^@pYb z_(LSZBiZXAF1NX>3#SMh++mUKsf|BApycUpB9F7tRSBmHzf~g=VK!oUpxECGCtMx} zlA7DY+as|IpwD6?%({&(<@l_+;*Q&+=14?yt_ZNpgzXU@d^$-UTbO!+5z5xe`^H10 zzL`f~>>&W-mv4AXOmn=_`NWQ%4Pbo-H=w@J=md~2JKr+^l$=bMg<|-j09`>S0Ymmn z88`oLA67ECbQel-$2U9xF!WY?F-1|&k%*@7wv(*a)u?WHNze7Y~iXH{LI?j+$A&j(8^XP`40LFkh(-J zu~weeps&Ibr40v>kWSLo3(g@!7wHMDE0(ps1Dq+~%QD+Z@;FEHrJzW^yuc)67X#Q+ z{u4Pr%02TA%~))dMJEd!Kzz+h@Etr)bCaslF^i90q#lNy7UfConja5yzuuhB0&}Am z;SEH|Cqs1psSNbC+txtl(U-p(g>rb~&y39^zluFPaN&h)fL-^B1B`Duz?` z&Rj%R4a>91kCuiIq4V{6a)hjJuIp`f$d4lH9O!Y$ z-RfG@y7y>=btmocYO+XMffSZ?#8)F9RbeptP_t2(HCp`Qt-Vp!J_VQyy$B%o$Xv5z zD7Ha6448m4O)Ncy0_W*Nd*RWxlZX}WJLkP@;kR0;{bECi=5eh6hD}TV`kmnawhr~r z6JIMZwNO>F!e0E3nbnosA5mT0oGxL)@+1)+9W6AR>ClPa{za))sa@Hp4~;f^qs>GoEg)wr2zc8$-`MtpyyXL>1 zg~TEbM`bQka6x{rJ=!fV%+M$<-(8M11SZ0n6T}lh+qk>q;Ru}*9SbE;LShHG6oBzQM*X6&r?+?e4ir-99`6Yp z-iYbgpjQmV^?oJxF&fWemG5-bq1CKV>%&)H&kVJIfkjXvU_x|Ia*}tVsa!>@Z@&te zRB#@cqg^p;e-nAA(`W7^mV4l@pAKH-={-s*eE-T^6C&7LPBnh~ZN1lQxure;W^UXm zt+1klHxlcCMUq;D@}ZtXOZnS%69aV)zJU8ikgmYUs^{=<+G3s}kf>nVj%|H2Yyd}) zG18Hz7+U`SvKj|A-)b}(=7DmSks22X%Y;;BVCrf;o@T~u!TfXlm-Od)aDbK!sBKO)ILdbyAJ04`(cCckx`7VXDP%cK#*io*!? z4Xo4O`SkphO<<^>pDj}DwIO8vu%!@|alH7%H5{$q?3ChACVz3k32y$`4L6${06&J? zR8rg8DNq>7Ifz{!S+fP$QP3y>+@!MXcdO%%Hj#kF9IqOmeFOzS84Y#b(CW%uZZ%M1 z)+BUqdQZe@!Lbg&QIyMb?~2oE?#|{aglgQUXl^am=W0$4EA=Pc`TcqhF7~G;tJP@@ zc8fE&G?F}tILx@NLke#?%XA=^Uu%x1hvH%dl*u>m*whOY0B#oXkHH}LUo0x2x*JeH z5Ob4fAFnE2UPXM}^m{!;La8t#OXI)iB&z415YKjTX6yAxhDZ%)xeH>YVM~AlZTw8B zRv%ZGn{0?{Y@jbiJbms0l%BZ;eKIi`TtjRlcB^$rAnrb zFKp03r?q)K3jn*|wxep$tORA(-vleN=~p$6RGU6EBZ_rD8;a!yoP-P(Zin(Jjw#o3 zaxAPLn%$>R7%4%4mPw=~H-4mij=#xmt}1~P06u9|e2d#@Ky_6>Lc)X7`vf|d@4xC* z;8m3}__x*UJxOv)p!Ir*T3V!Zzm94EGm-s;;vcFD`55T2$PvUti%_LR1K1PbmCZ2! z-7-&rTY;GJ)`x|(Hj z=XE^iYgY(5JmjUShUhXuQbq)j3H!4R^!IyR<+EuqUu9J*|7Z0<>v?sYl=%4=s`!UR zsfcx-wR8vZ`Rg~ zJ)@Oj;Leke@pB`E3WAcMhlfWwGRXhVW&zt4DmZvF3pAusTGxDBrflrURjXO=p4$;m zQZ2~|`t0X(+Vd^uVc%1)6S?Dr3~tW|jzVfY=alF_)D;O99zNdNcN4Pgempe39w)O< z{Bba2JvdG#pkFgqG|QbNAs#-1t3R=hM+hIrj9TV?6wB%>3|l8V7P;KYC>J5nT>k^q z=eg!H3_YAJWo7NWYZ;Gc0*K;D>e`;%?KIwxeX6m*vCZw5eH@R7tVE~X@suWkYe-U^ z)s%we;Y<@iL^UONCxxjzK0 zE(XivRPF*@piW}v3sg&eQJD-=xojC6La;`t-0=41Ym>SHdkYjvX4HG_Crf6$En;=b z0Ux~9Gk`@X3kY`Si#;HpO@(G{rB<9T6sel>IqyY(cp_!H^|(`4TrMJ<hZbHXCw;drSubJ32qZP@0fT6M)zR+q@ES3u523e9uaUCEJ<=5s&D(5|30RIh`O@?J?HPHc%->i=L@%-k2?i+xizjJ4^*q}+wYgJ-<+uer8ZRJsUe|C<} z=k=SNZ!*Il2I}YTBOXd(F^D~yua$PsaNYO3L<1>9qNwIP)c(CT0WSXRlhnoCdrY7`5)ZS4yKz+)tfc4jo~(9$1=e!_10S*v)3bQITG84l`r2d zFL7J1h<`m$==@)ly>(nvUDrNNrwSq=lG2TIw+IX=-Jmcc-CY6#Qj$Y=H_{9-V9_Dn z4I&Lg_wR5&_ifzo`+lF#@4o{xbIv|{uf6t)Yh6ovo+XZ$La~~D-_y4*d35O9j~6}> z^%y=`P@M)2_c-FI-oYOu2(|pEt7W&Cz-6Z7jVPBxD0a^>2QgA`?0snQCEye#4GSEC zc^ zqPT|vQK?P!DcSX!5m~Ch!-ZP#ui#D09mKR$H0hBC%6Irch0SMhx$nJ6spJTm_~%|b z`an#xW~DRQz~^#b#xJJeb!B#YfD}MTvf6fi#R(L;DTvy6N1T**poy^omM3I<-c4=Z zoBWCSZE?>`#e=S(A5EG-YtFnOk@5#f^sRi-Ed(_xwpc{8Egr%cqqQ!Nfo)bSd}_?6 z(VZHdVB{{{=(1xK_X^yL##h*K~1NuwILzp1pUf+owy{QkqzeCqD5+02fW5pQ^8mvVCT}qCopI~C&eb78i z$g11B>5S&vx;pk$u639L@pn(V?Y>=_Dq-Ykwf+v2$eb}7NJZdUvYpQKxGf)zD||Z* zdC;e;oY9{|-{WhTS3RI_rq`FkB?EB4vIIIG16A^R;EkF{(;Xsg)8s~JhKjUvpdV?Y z**4IVJ0nHOGEyhhLhC6k%^yyW*MEq!)6cJM94SH zBbV%bPs$A(6d{^!NoPLm4(<)MVv+kxOt|b8b|rjf*GTTWS4HrpLamhoBT_C5t*gc% zoh^VEz4sc3$8-S<8hzL8cS(9(jteq+Rv6X#YSTCz<1xd!GNP5m%;U7!H?ep4j+9j%WLAnP{Rx4 zHI6tvpa_l)SdjS2<@U|oGMy?T0!^`-%WAL1kIPck-}}IyPWPX=*Shyu6Zy?HokZQ@ zb#jm|u<3?H%0T$CZl{Vx2Gv2$S7Rt@C%QZ(CE4b0E7d=p0nIzQCpb)UZMJpuVew&Y z#ghaX~s*Z)m`$<$n6QHuJKt$F}KtInTCrc@A}F43Wo&o!@tF^yUrLAvdqBYJ4U8&Eb=!+<$PJ^kzwGh7JPliOSDr zW{=!Ids{?*2EVC4*jdDeC9+e%ZeVSp1;CEXH}P3qayMW(;d;PBwUj^ksWA_HkPpCy zmAGQdM!n1OT+##`fLfj6(JgcGehRqXISqg;NY~j=3U2HHr1%9)LX}*)oiX zuaxM!Dh_+~UTLFtV5v6jFVvJDNfc1-ydiJKC08Q6sF|Y`X+!sD=+r@3$KB!g5S%#D zzTCo(XljS82@ZF!P8IX4cv%XcC!;#T`pw?C58#Bp7isH4=tW;6_<#~}wOlp3d31Ri z2GRUxzRA@2{c$7CSBEKV_E`u4k}LJjmWro9AMGU8Pmfhdz4@&h?U7xj%idG+IlUyM z+fUP~AkuCkm?JB7ep7bITPS@W1uKI^rz-JkM}#~k-J<2Ka@kaE1c{KX^^ARwx5xSr zQ2Icx@cwE|TkT=Is!F#+rglJESz4Zz$LeZ=0<5+4Qp3<~pLTiw)6>QybSxGI_vVm# z7iAyU7%Po>4*-Oh?p5tp*@#|B#29vQNH%6+1QhS50o1@kA`P5xNkzRnKYVTi6~`52 zn=bTt)fg$rw_fwH#)*b#F8RxWmJ1URT3;;aZd~V>{`|g0=u{SwDDEuwHQF&fLodj{M zcT>qln2>dF1?lo~dgE~26OlijLWPU(wbf5C(wcADi{H@UQXbPJ->4sF`LCuFEGJ{vh%6jcQ}scR{t z)lIBAQlMT!TUvcoSWW>^VC{I;de+#5Wcc`8`0|6obJx{Koa5&HiVHv^qjA~Z*&286 zU(4_`-B}mZDpJ*`o6u*{1-nmYvogjMsMPOjt+wU_)4J|7$1wB7nZbZ062kyV7GE<# zwDIDL#TR(lV;%{TY|r4yF~*RWAAe~IAXOa$?vLM$H)-H!1Ik7!uVY(73nx^>rJvKL z0|*Gf@s)jxLb@z+AFn)s2h6M=fYduOFv^5qu%+mH7~^WsHdej%dncX}~_o&Fm@%m2p`- zU#)1no3R?yucvEk?G%{-gwoW+v~{KF?N}2)%S|0=T(&SB#}c52w0kd-b7d35$%u{e z3t4pOST)A)9Im^5a}Q^_oR1BYHSbDDzRi6-+?AQlkNmox}yt{vow$C&X7Hgsq>cM)6j3FmoV-iW@jf3!Ist^ujH3~ZeEs@5bD2rZ~y zTkL`Hp}9Jwc{38(OYTniV-kA9;t5;#KOK1(!o1`l2@)}C`RM#qrHq^{ZE12Us+rlXQZutnj_}kBniyp`(jgWNyF%1oPve8HfQ87?7IN$)Z;l6u^ z3w7rb?l;$Obb1b#z2;{RM*a6xiA&yM5gxBsTm71(cQQyo|0GDSPPJUEssQanpBC9IeEKj-N0 zi>Nm!4Rq>ApUeh-eS%AYQuXOv4tZ7p@s`NSe5|cA-8NlyZFtRjyqjX}LxK zXk8FKe3|q=XmSB~A%G%@5$*Yv74UPTjN;L|2X&yOAqTy@PZL#xq2J^(s?GFOqvSOr zftG0!`){WgAfMuyw-TpPj!vYD_iL6vJb3qk!VUlf0&vCqF#xRVCg$|v1&}*e=mi24 z7m5CZ|CSU=`}@Fv>wfE#$Oo}O8r#_PjVptUvlvU8JLYFM9#2(R`%y8 zlu_PaxsIR=Kk9x!7WA)82v9DnOVS=Qcb)^N>)-DPP~d)e8_H!__*6cSfc_uM%zy4J z%XGW-n6zTH|Me`{I253JDj5GcEeOIWB;060-=VQylV?M#<{bp!A^`Dud3e$%EgG`EDpvMM2W{I^G29_L4x z!GzLjrn{d-zu`k#_a7q3`}Q?#Z4{iooc2X;M(D2C@%;T*>2$BNzA?dLo_T!682RtL zlQfQFR_!er73(|P=|`m>;Ls}eFK4|UJ~jM8BmOnM1|dw8aJ+6+QJqlwi&gPNf_tc= zUD7f%Y;n)chjQb$1RT1Ph6`4Ke8n-7-3_6iKGZ?p%`Fkgn$Ush1maBW{ER2-oGs(_ zOJj|Vsx`@Ujy6HeodwqLl`dN4xn_s;C0b~1wai}?27k?Vr4*%`o4p*+^G3e(%Bt+3 zQ%B?Jb7FAxBCk1~Y<^$p6U6{3hNSDB7}}JT5!jwDK+$t#Ilfdn8NY-HphioNva^|= zcpj(5D2f4*%BX|2nTw73=?4a)*Qvp9#wO7RrE38Y-jtm^ffs`de@gI<4HDwwdv>#F zB-PU-2PJ7Wrv#M$-+&kIgR)z|OS8bMROS6LS%MvChkkVuP{fRC|I_!h3q}Dcl;NAJ zOs3P~KY!?UtVO;xW^jJIxogXbdGFqBP()ruC2HPWg!adDeTLJIZ-cAqKn$AJy3vX4 z#lHKu5?6}#$r9J_uu|>HY$U>0cI6~a!nHBt*;pw=FoYrttEeYI zN*(H`1YcvP0Vt&_mA-ZnhB7pK4~S#Xpwm6^5^(s=X>+NRA(|DtF3i|jBT}x|xa_e^ zj4Rx$p<_J&0QnDCo=Zd6V+5To%x7{#@PW{oiPPJGizRu0(K)!|Kyx9@C!y49$uUL9 z)#f0)r)ZpmE%#Z(Bly&&?HB2>-q=6%L^f^I0*HYjmgU5AaBPh5F&qB#uGkkrlwT>3 zXhCsNWKAdCG8Dc`FWbYn<8Z1f_eThmQiR8&-^^RO23e$Ib_QL>aUjMHNM zPY7J4l$X&q#cwytx(R*r^iIfci3y+%tD&uOBJC8{D;PzpwZ%P9jgs+BbPBH@c!U|~ zBIF*j$OgrGj(eD zRszKEFR`k1`tEv#woPXvsy`|_g1a(5N`c$jKE^Pr(RD>V&bdRz`{~#n1R?43>QP}$ z-=m0O(y(Au$&3P+z7?v9J%!}^(}c>xv2}y(>jJ5|tNi@|3ory;qp6-;vpxc1ZfY~g zq;-dXR|6~An?S*V0#gJ450x9*DHTjuP0)FV#rL{aN;LI2`f_hM5`Kjv1_Hps`c)mC z9U>>{aeyNZecMEFdT!vLoDkK4iiIC{OeQFs^?Hi;bpJ7@Io#7ntM-->IL}h+U!hN+ zCLf61Ye;@oM+Ebb0#dS{LqqC!NbFSh?r@!!zqJIcyA1F`J&l$uR-al^TMZK9cp6H0 zCtfGRYfH`R*-ypz&KpUENt)*a5JzzL3uBe{BGwPuoB>#CZ|SqkOnq!ON6Jbb=v^NO zr4^uj$d`o({)aGkxT{$~_LH$&UNdH@T2CHgI4UfW_f%ugJd_=CupaAyvt$s{bES;bjP}Kf?T*{y^PZh+ z4QTlRyKqxla7Xkjf&IWA%Q5tk1_+L6!SV?)gSN1u(lEJ_3*DxXJ%Fkte(a>y$^Il_ zR{{!8ha@z#mS~ia%OwTwJ_n*t)U)%)pR&*2XA5s$nB{$}rfl3LVrkjC`rM)glSGve z$;Q)uL&$D8LkzGxvJ92#IGRsnW^qxZ9v*vt74RA_Tf=+*OO4+iol&?bF9&S{Z$=|S zP~$uqJC_$u({?@%k{19R@_(V$oHVy{zktRSL#bOmn zcoSJF$%jj9!=tx-ZDIv7W?d|UiE;;*={nQXguvf>JG--#(wab<=sSGwD>(h^%K&o( zKtvV~5K)3MWX?fA9B*YZT3K4l+7b35r`=)x>ZIpRS&4!DDh^Tasn^DOw0~pi`H@yx z2dO{-n;HWR?hlm46EE%7CUo$2G32>st?@@H)l9pr*UQpLn98?#O>uk_e=AFIsLba# z@MzcLi`5{U(D0;cF||MtmdU=?85|5Hb(j>{d>t9~qiq`w?#Yf=lYe`mThHr!i_l^p z>@(hPJ~P5Vi+T*Sj*6v=Ah8guN^3r!vQ%x+)qn!k0Jm_d!;M)zPy);2n+uDq+;KTU ziS+=QB&ND=--GUMtm}#-g!i-%8F~%ArEUe10i7n1_r8=HZ}+iIUq4v`z?XSaO?6d3 zQ(?z@zJ#w~>*{A3m8#M@m)m$nhP9!>CtF+&d%LZQb2mlV_u^M~t`u?&j)$=Xe6RhF zWw@4Acx|V0&(vKEQ^;oT813X;)WCoX-vCH9vuvMFPC+bKUw?Uqd3-&7E9>fcD240O4R9vsOjo^#PB8?|L_+XCsKUK;dFRB%xN12*{4b#}$lJFyO7 z7jrjdgR;DIk5LogOB)N%e%$rG;V%TWm&WKo?#D}Ftr}=xf9%mB#mcz+h-y%lap;xa ze5N7i{Omosz*Y^saoFQoo_f`oe>YqGc&Pu-M(5lWdDvH>jo*mNUgTplz=#eJxEDj2${sHrw8(7yfAt zm?+NI3*tIZI8oBog;&9P6nb%Ox>)odiFrK>q{eNpzO zd2V~^`QQfjaBi2+t=EJbLmUAK_YDFHB?2PGO2V5!Jp_v0q7_qx0?Qk0TK69*mev(k zW>Rtp79Bg@hm#3L4&Mp`uqavetK!b@JuIBBP2L%SVsDGsy)#VZgzT1DDJqK{+_9dk ziw(L*VY4-zQBT6{hS8T)9~B_5s|Ij62=EVwjoD81QW(#vn^1S9c~xDl5E*4_Z1O&= zhhSnFZ_iXGIvpf4ZG;10827G7L zR}ebh$&nin?Qp%S&4*oG*muRRj?_*|StO`dneIhBX0po@3-U6j>XC`$^5V!gq93Nt za7&dGbWsfhlcI9CEroDc`1DQ`ynf=F^kSb)ZS?b@j1hT)M4fFZb3`l|5me1iC{k-Q zL*BiUCv-La3SF+-B~v1k61%VsZ$5gq&H$s;i~C_sSu z!eSy*v~MLPQ*i$we+=2L47ok69ec?MbBh1}U1P;UNWSLN0{}aiuej zU~CQJ99F&h;p4O>0ZQleIqws)*zKC6hxh!u>?6rVK#D0mIednm5W&E=g>9pS9TZx8 ztX)J?LUp?;#Edrvuv9vQ)I!cihvnLleyJO;gYS2bbUWwM&g*=xbkD8(+4bkn*sRYz zMfz^6>zFzi_{i3$x2Nkzk9m+S?7yo!Pr2h&F}t8~Ly-Al*F!kBS5ZdyN4#}4D5w_wJPIMsU0xoiu+zcP1Pj9CT#;&Q z6Mi?(ceke%2Ojn6j=23dxf|O+UGvpol((X1r_?dm+|9%fTrg z%ffOUOj%4}UC+qw>)5mA6*p=b;e3BnPR+EjDi2@tD;s<9Ck^gg$wFrDJt#oN`tx%D zM@6k|m(`v1Z-!S*XI~SBe``04Q3=O8cKtCbv_BJMs^wdeTYYXn-tk>!ClKOHUB7_Ea}epVJMG4Z+@<#rA)Xb;clhp9DS zn`yVSC5cR$t3FbOX=8jBGLaGC)t|*=rnPDoz_ujywI6!3o32;y5+OL3yOuG~U(R*7 z-J_!-g?3M+M>&mOuGUgBC%9=^*H-NFjsl_Scphm&b=0_8zl^8Is?^HR&Gmq3YoedH z#DfN>b>SP5O1ZT7TJboPOw%;(vWn!7IJb%m%3_>8YZohT(Us7xixyXA#AI1OJGoM# zS(f|ln~H9audnQPahK`OUeuAMD^@F8J2S6q`L*r$77$ZhYu|8{3^=&I=*o1vir^b{ zfxF6RDsu7wP|zO4sEErGBRrnh6DqS-G`?Cs*Joxlv`Z^tu3f2(cGt~Q25xTrWFii4 zkL~IjO_jkKkg(U5Qb6 zUHGkI+!dB_rRY z5f4o2z`xa}j8`}k_?h4wzOY?077onKeY#xZ^T`xy-mE;&Dl@!aa91ZLYq&q=GUc(3 z^7k7}SqiJ8x@V?pctBb_WM9nlEwotpnv$ea% zo#J3^2aPQ^JF%&89k5|rv-3md{j82F)jEHTj9w(92_Q)eK|kbLu22uY4-SZ37mB9d zc_geppBB$z*h^t^lUZ0eeR$YVX_4mjhK$1s!&e(x-aWrP0n=c${d{zpceHHM85|C2 zo=wQ`F19$AkCMgSIoveEfEV`jUPf&``c{y@L;%!x&>bPl@urCF^1%Y!SbYc%Rd-*oYm`4q8S)oh_#lN^|x zaT>KV4Es~f-VtGl5cx) z`al^zUu%9imSU^I4wI0rYEEBapJe5+^dmSE0wve^(EC=lI(*HjG3}y)l-aNBJtjzm z&W{fb3C!+t5{nfZ-T3Kudte&0cx~QCh9KnH-`-|-dqsmWT_pIXW8sx*|>++9wy)pUa7UYl=4t>8&E8Aof0PkjvJdK8G`InhQr< zXR3dQ^T6BkKvf<_gY(S$YKrG5FsM#Hy{`ierX`0!K&(zYxeWM4wPb^iJ359h~C6bRqW7(4M1kZ;R%yY9B_W zr}-~I4Ok222k_?LU6r_e$Q^aAQ$#-`CSyk2Km>I}?@BveT!v3b-zTx~-eH}>o(Gkp zMYJ`y*6Dk~JdT)-R`&IB)%Sl4nkclptpW^V!C`eMTRm zQlTgrHJO=|5VG8DHP7-d#GBZJtl#e(BwGx|6|s|WK@Km*&U>mgk1f(5`*~jcj`H0{ z=8>xU=cQB$mn~mSynXhrJ!vFZ%FOdV=i7C$6Eo+dNA|^NOb?itb&vaZ=V0DQgVE_R zgM&UD&ylmPGnSQcN;cm29jWVCHk{oNMGOYv}RiwfF36(X+?jr)^8Ty-u!hm@sH{!q4gW z@Va+f!(*GgCLJ`YD%``2N2Nj6(59x6ZLl!o*~yooiz5Ux&+fAbk4wy>T^(2Q>W6oK zKN8MpjT^vht?zAI^fCFrrgNfL=YC0NMP#TpFcr|N)zui4L387=U0lqkeOaT*oYyR0 zq_RN?2re;1{K+X~DG)7Ok1&T&M7olw!9G3m?D~UZ^yMcN%<#Ki1+b z?M*qR4XB&B41m@Nw`Np0n&Zi@*_tP%mXxtgInTnp!CPZ$1u$UO#>j4rZp(Oj)CatDpW-JL04Vz7g`_5Oj zc-q4*uOP{jU8C?4cX`y2rQ0Y#+fn2j!*`7kF4yXI$s;iv69lMOGVflWMn$sdB}$ioS!)XrDJJ(#dImCd1P;VH!ov)K++V&U z+9kl&*1gQSy;SP{x$o)FVw>NU{`C&2^X{C1$O^V~SrWQR4a(iKPj2vofue=sjati5 zqCRXQ78USEQ1h z$-AxiMC|EWINoCD=rT;3dhdIh4PI1u>O9ejU^monu3sbqepZ%H@4#d}9%6Or_R4f`*hgBPnk*O-ISM!4J)>F(Mh6am-V$ zuT#=N6>KkHS0oj!&zv{L@=nA~Rmx&~^`3=4L7`lA6$tGS5asa^jtseA(Xy1yEy#Ez zQ=pWlp5(vVe8JWw)gB!)@9N#NCBk%Zlh(0q(Cm~nwQ{<@Otbd5Fa9K9qf-oT>A{Go zDG6}`2;{=M3?kF@Jct+{t@S>tUG%1;MKN02pU&+|+~-O-vdXEmoy>-go@`oHG%EmI zi?XzgJQdSYP`>3l2|xPF*ZVtpnWlfF@hdpUAk|tH?ebuyq@_LWz1XPC ztti_@z+qwStME4pQrrLtd8)QnLYBP}ESx>S-T1oQ6sHH-X)DS5oP-(&-&ubEEc0~1 zQ9>*xCND>TGKI5parYJJVNvd`1mSbdte2=)Ti%MTho256r{*+?tP2R3Up1?7z7tCX z4Vx93U+n{C4O1p+N)07Gk_ze(n=Q209(lVHY#QA| z)Bs1VuDQ28^(cV<+wCE?$wb+jEoNKzFd_ickIq=qlLkUquoJ1-N96^835-t^4SmE z*VO{L#lxdo17Dh$jA(P|&IbwC#eSzYQ8wQ;A>ULd*R||739(R|HUiQq3f+`*yOu@U zVyziv=owCC`7ix+#h*m@DyKn9bE%7?Oza!B-#1*+*J=otf8CspvmW8xG1 z3VTSOl&IFC`L-8*0aKXBf*NUQzT{=RJg=QtRYG9$F5rzIFq8kFdrb5&%6&o4rhD_- zpr8wJpLVvA;mr zH=^hd-;IT>fB>ehp-Q+C?EuTG?az#}e}{R`dC<=;35e=Nxw$T|D1i>(>V5KApP)TDa5| z8z>}lcdb0Mg&mZiJ1l+*#6IzS$5KBo+6h^GbPRtXW4YW;(*AzvJF5u&UNF5m*e!Ez z#%j_z+Nu`TxL?hrRaClD9`88S2$r`=e{_uMCrlItxSJV-y=f753Wdkp5drNikA!C& z6V7mO2MtrStLhv3fbLZ92Im=`Hg??)A#+4L9Xg{83&j+h`7C6!fO(EHf1OS0f%ENE zTVd}v60J(--HAZ=y-)M02GO&07h}HBW%Jj7=}3pjaMjqGzZwRXr%STVDdTF@)FCG% zw7`241OARy)jLMFdUBXPwyBu==!)z5Wvdzg2fXe)7K5T@ig|_hH5<3r51_tIXZ4cTir*sd)5=7< zpYeT(H?Czh8!d7)gFL2P9X^sE#;p5+tn&S$>JSF8E=A2eoYn)P9s58y>xwr?(UR|4 zj$%?qb5~iXqE+(>)>mJ9?9;an-G~e{|2>*GQy~rmOUf0s6ce@w4k10q({{i)?Bqsf zDj;G9>xwQuy3Fvp7Pw2o;V<^^nHo0vjT|iXTRq@`*%JqcsMlI(OttJHY|l2O$hW(x z^=Our;@CRF}`97c+t?W$xuJKh*FR_kIf=2LGV`p&VK!IY; z{YWtzFA*@p91xw*V?jUpNHFH^<6&x?H-quS|S@cLEp+ zkhMxBaueuEEAbX2bXwy=Vi{jcyAipT|3_Uc;U-M(F^r?DKKoG7BLcA-dJv$ z-T>w4StR4OQP>I9Ztu{`Aqa5~<%Y|Z`yLWLqz5mpt&0_Na!WXGe*5+!GO4s@JsdES zcu0lq-mGM`q~E)DToc>D`B2P7K}i{LTH;q=TKo+%1takdd4GObyvS;TVKRG4idmB< z$RSh?5@@wc8BDSwjQ~=T7Bw^XJz3iu`9S>p8AlnVjO1b{3RBH^Y?Q2*Nw1M+@vxr0 zfS!_AWAd)#dL-?n80LP66Kh!GPL<+zCM7{3)P8xcZTQ@KwcSZv9KoQ`iLnQM2WrnE zf3XK=%0nxIG~RH+`~U}lpu25}lThO2O8{j_e9@QI8U@rD$_$8bMWYW~&o=0%d>H*Y zJ1YftN{Xz=+eQbxIq^ZlsBSq`PliC`OXm$fRd7`3F~RZ%rE!Y5208OHn+Ayh1e7Xv zN(e3+z`8W=$KZ;i47pF6q!L30N$88Xp*!AZ{G3O1z_!W6K0t91ByNvu=j^P;*YNDw z97#P(eR-38wK2pwY5ZnvW$f&w*gv9!uYh=#vUD09u+lbv^M^BJh7dn~gJ)#uTUNI0*Vz050?+mJ^~;S+oocdb8iIq( z%6CbG-u^KfN|h)S+_y}!6=?maKcBJU*S4+4*ERZ;2j@84KoT&>CMGANtE;i4=Q-lg z2KtysL2#(2+sb#FO7?#Sf16AK-UC$$q5cbDLjnr@ulaD$qTD#PW_tc9L|C|)fsxT~ zXRals3B7HOQsoN@DBg+6jNYvAE8kx1w|uVQ;OI*4%67B*eFNYD&xXg4T(EsX;=ELoloSq}$I5wodMez4=PpKg{C#0T zoRN`ngEQAFtEmYcpy!|-VDACU7gaSiMmoA=CiTMt<`-FX9i{3}ECZO+plOm%erW~! zEX+A204{g_HSy1dbkISNDAY&$y7Tv|0~=j}0O*$T*gX?il{0qS2V`abpU!;gCdH5$ zH6}`2#fnvHlc)GO;dHDz+!>e58u`4|)5EDoX0Ln;CTD}HRt2&ALF)M?YSt*Ovo`AJ9$0_bqtPYGc>*?W?AK{Lzkrb`X43qmu$CPVVhnNU0zWjd;VI!|9r!b3W#d%k&?+r83kfF ztuLYKHru5)Rz2cDE>vf<-57R75NP6$MG5%ccLiZmFO)?_yPE}MZB+nI;Pvo4NoRk5 zIuf$w@XP*8j~Xo_kdl(-h5O2&sMC)HA3>k3!sqf;$=&w2OhB9!wUXu(q7y4Vw2&1! zsAc41u|ENDwWz0@Y|JLu93;AfyW?oP3I!cbDI-Y4qHXkBmGaJnBPqn5rJU&x{#@h# zOei)b2EOg|k1- zlsm%P0!c0(Qr0(ASxY(McXjivpKMyTJra0n3PB(s%A_+CQfE8;39)*Uj##yh9Ovqf z5hJPr+ebm<3HqM{MnKZRkllMmE}AD_(pykI>+vn*K?eh!_YCyyTNIGIyc+G9X?XtL z99~-nh%0f~&BH_-Sn9$akQ}#X8w4RF2FhZSkt-;b1 z;OGCLWhQ97J~4%M($A=mnnB}exf!oGl4Hi+-{6pCt;bhV%1|cuyZ!?tbqjljc_{75 zy7FHl3@89DKRU$}wHw5MQ4N{7W^;@ulR7d{rP;dr9zfUJj;~*f)(_ThR;>!(J8i6B z3VLtQ)UJgW0nLI7#>~ZNxQehM*?F#lZ5#CGy$B|T#wOub+Y?pRWx9SAM72oBjPC=C z($fu#Sr)=GU#O?uft>65pT_`S^5-60MVhNd!Apl z_AsPZSL>qG=W3Qs7jjepzEbk&4q(0hzV|vM+xmMt_%>T-swXftrT%m7VfnvVY%1gi zEg}wYj@Uj$Ki+1;C*BU+1xer12p#8Y1|^*8QNrY z;OJm)Pm3;tE_(C&K!b1%tWeP88p~4NW07V0R8G7*UNh&JqDNr|^{wU#pDiQP10mDV zCZ)%c%J&b)Qzq;qB)$>nMb$T$1>OLOn%dUXv8S|&&xWw{+92B3=-ba(7gs`Eu?=08 zxfc`Af*+Iv^;Uu(cHhP1yQROL}ewZe@+dr{AWK{7v54UJiQ$9kzW*n;DSorx&p3QJ#I77k zz+n41#{d0Nyc`SA5hbY;f7+$LPWzv~e}h4Z5jc zCZdcUCX;5dg{9WvU*q(XWWjYglYA`dnJV~y4O>M85N!<=L(Ek4|LZG%51m00iPubW zvHz6nuy|SZ=Ue=1w1005adK?R2TYNEm7f1)&?=;;fEn_>La_Xg6yNVF_4lH!R3iAR zxZeq%{acp*uMzky0V}I+?=@=Q_rKib_Zg;)1fJb(V!ZX|&iPG9{=USQ0kml7a>;wk zKP`d(IRtkW;MwHJCMW;p(V6^!lxUC|IqP%%muc@b0iF$i_^>JXf1l}}|NZ(LFiVzB ziUd$2_cn`7+OX~j_7iOh|aa@kBdKFzAfOsIPS(r{$=2+KPTz- zlDx1jVpEHew>?6CcBarYB<5*h|B3jO{S%BtY#RFcJRkv#Peft@!#eo+!_3ax_4a5>=FUj7{~eoou} zK3MVRgyOix46Tg~y#MbT-a6nIlB*Ad?_vL!M~7Ac`}tKxc}t7Pkyj0l$fYcMHIF;j zrbaQoqSrO;hLG_G!8d@tezY|j43PLF^Pzt+{4b+yL&ml9Lg@J5fq@BYXo9AcDk~^t zGQW6nxONk->ZRY@)rihvvb!LD^oUzEShscwCk1C+D~oCr10&b+{-V?Q2$%f}z6%MN|MKd9O9kh_0L!>^FEOFpl+QOjD`&G&hk^)- zHCxakGERjRFSe)w>xx;{P;($p)7aW#vXLw>Fz^*KhBN#NM`*ft@Xky~Rika`p91?o z#UfCV(kMXX(b%=`@wdeQ^bspYTM7@0Pu1#GxeqiBE1qU0un!gvE_bFRup4k2^~uXH z>JX`v@LJDa@?BNC^3_@sN%r~c4GmHU1I9K*v2Iz=)O^x#$NIp0f|`XzaWdb{bLvb& z0I?;R*Ati3(4ehby`g--W+D2)h0J2MP6WcZDUXSX>2y6q3{0c1MXM^Pg3e4ruB46l z`9SL-XPM3!z1*9wuC64H6B5~L2vc;938v^X((UQ$SeNaIS6gCsopzy(-YGsw`#HL= zuBAYSM`ZkOy?P+j=$`1IAuuU`cP7`dKsm+rtMGplc3dP>8->D&iHFqTDA~F0?(QYV z?|IpKgj+yG;2nLKQgTypL9*>o4eIE^kP8ra+ITKhNN%P%s_xzE+v%oog?1;a3I?Mw zk&yc!17JXX^SVb0ZYb{|K?+7{)E%6i^BRvxR9pR|G_>f=%&MxnOr5(NfkOMP!H>A} zBs?Ca-CHr>q9XH^@86%13A$u?96c(wmKPK#IygRrbBdMPUx!pGmn>*LLLXXMUheul z9ENs>v>mass$KMt82SIU@lfS5#Z4sz1mh?>@Mt@Gs_~t-h7`|aiD6+x*L~5WTRLKD z+zIubRwJ*OR<{+MH+eE0&A49Ob>5m$M9nb0%$wvYa12tTitu4|tiZp*2_&-$C3B3xXzluSVm;wN4G7zdS zdK2c8#5%mVsKr1ftl%u-aEeg7`7H!++v^*O;_74TEH5-op2&c<-?dMRHJ8UMi8-fG^?Bym3sNm@EO71fv7-QqFEJ!ifHgYm6=D?dU_n`z3VrDru` zb&G_7d^^kPN$#z4#hPt-|0Exr1yeATEhumInR;tV4IaxqG_)(>ZFlfaI;yQv6207+ zs?-4MKu!*Fc6?gi@f#97@(3a9OOXOLT|Dn+&Y!Bv7(Z(00`#d-{WuU$;Z8^Ne zAwDxAvt=tx8AfVHUsh4JbdVcF-9*XU6TV`feaK2DIK|)OwC@Fy!#U^QOH^jmZ{Hm& zlU?>Wz$BJ025;6C$IX4by7)2l^#K2&dIFN=ILB`C`sJ&8oq%@2;dwohXn~6M918bY z*)>XPA&slr0VLv@DuO^*)FB3a}2Q)fj=rfTRwVik?EG#_QIKqAOG@U$Xiz%r@Kd)HV zjUJ#rC^wX#e^0TzpEfhXspl)(ww@JCk%xAdWbnPU-VZJErj3=591F+Yp*-u^yXXJd zJ}Tj;UZ=J}Z&AW+)xv#I?bZeG-$h1J^VNI$&ac_=P5c=Ld?Q{-z<`^ik_?MW7iuGx zQcS7Md=`-sZHger3m`!3;Kyu=wE-K?SGz`UbucOEj+jrKXUDx_L+Ure{0)bn;>HC` z7z0UJO|`dYV1bQJn;(}okJSAN@Prcq;wZ3s&~Uus=IDBcfOCv)y?Md$QSZTxOv&|S z+9yf&5@*9ygQG8r`|8=IaLI$QI~>Whv3Acfp;xf{yT9#S3xz ze-W!b{p+eW>HZcR9GBqkF-2K?wgtvV1{#%c`P%-s$@aO z&4#OyMO#*>5^GLg{Jn2~HG8HSIX5@=&fZ=+U}3`h(|iZz!Irdy5J{6DWdG3ZlAu?^ zV}1{(I!wIWdErvBSEqAyv*VIxU*L|Kf_ zOZ=D{m)oft)+B4mjfUIW_BCtAC1!mb-WAaPLHY-t8>@OP08~C>>M`zGZfuZB)}|qu zt8;h1`alxBV>{2t$g~wH?wEH~lqjwtV{V^<%ASAZS7q;Tv+BjI=XhCKc{;deFcbW>ZqeeDTaW~8I1YC3ul z9hz%L5o&?R&w0zLg1hDBwcH$>LK9N3;=dC&&uUK@CP(_o(mJg*yqSW;xxTN8XUl{& z)ofqt_!l22?ja9gMn^AmlXcdSuMA%KPnFO5|8W}gpAgtmc5V^W{!>eQ zP!sW1xab?9L=%D_(pD419Q?4?dP843!fp8TF|nYrxdZnj4}kgtOZN^iN@;vtYwX3| zPu9UgeUEUpL_NLvwb@rCYofh@r29NUguL_`*vm6BCA91>e|Z6j2nh+z|M>CJ-a|#- z{R5xl@+*5c2lcw`MeV^1i}5vcWuB9i{TQrOAomXl z=-S92I8|`}A#OTQmJO=`I);86;j`CQ$ByZ-#_DYT^`j2&dk3_8Rb>$kGywoabY|7B+MD}iWHR3uc-v%Hsg|Rf0wy7Z zS61SHVxF*2YFdh|m9lna2PzX07WRZ~Q#U7~wc%vFB-VjZ$@?ZIp{&uU@)*FHA=6Wn znif$n-Ej3aq6={PefC#>;x$q$1QTVh8c0YSaUQv(?I*_0DI_CeV%D$u=vLQ%B1g<)1j{ER36==pMR=Jl1t*VU}Eud zGn8)wMwe7z1F*5Tr9v}@aXP-9j!I2SQ-jry=y~WeMc=plUgSj`=pns6K2k}#F8o$9 zUq)U&#of)Bmy&t|is zW|bYB&ENaEjD|mGPKpX&t128G&Yuh&-yE$)rHJN<`owu`sf~FqF;+5~g~a>AU3yhYR0#=m zjC$GZMMO%Nw=~WZwQQ{fkcSZ+DwR89*(~k&pdM_hb-j9(D%s1rQUSs3uR^5$BTk8k zpiP|aq2c~J8b7}siZCeW{_}>~$#yEE#leDxzKZgzhxQwUPx|Cy(~;PtPK+wL3!k8< zT`*R9E3S`KW}0af8exh&=k?2wi|fBQKfNZ95*q80-gF5_h)-DlY7D6{QTTrElT z7zwPIMTF&B|;GV%6N3pHD|p#M42yoCyK2}Rc z#!i%4#U1cVBoquDt5_9DOC%Hv5$283vPp(~&7v92tyxJsuX0^}crI|x56a&(CN`lF z>>ROdQab^cMUH)v#~;a7rosUX@+2h4^~?8mo|LZlK>FfiUCgDFk;h_U9;s65=Z8jW zXa`sS@qZzyc>rlPLb)06ub77a=Esh+R=`qV<~N8BBMlP*)trdY-M0f3(MtpIttF;I z(&$)NP;e!GizJO0bUAt{wm8rhM6y@gyKR^6sYTS;Z&XnV&d(ZQ4zp*GG5N|()t;$+ zVXMUS9&>zVsY#y=8jo0;;yWL#Y_-*4`qd@i#}1mZZ%^juLR1$AKkP0IZ2&*?Wndpz zX3|F%DM)2>xF;vQF6A?*0dG*%R3rD;OR+BByU%7v+Y`$hUZ7WRs>6)Wq)l8BDRlDl zMvdDkSN-m=Ha&^#v{Wa)bjEOq@I^8Kx7n~xqt+Gi&@i%Rc=waU>y3C`yT!d_OgEd7 z70n z(mC8(HfLid#@M#?-F4z{n$u} zf-IbrVY5+33>#unLD(`PI5NYQy)YA_)@bu5>s>VB(&r-#=gPRO*T*;`ne}{Z(#7(I#Q_y!5Z>PnT2MxRmh;+$kw3v!HmSrNi^(WtH4@ ziFbL#m@3CC$nLF<7d9sAo1Du6&A>u8w?_Hu&%JJHYri?=;DN9stu^~b71NYRa-)#jUHVHr7o8kk)`13t8*a6(WZTOj&4 z-1CmNQq*A4^#X{t7isj8I+@4p$JjsD^lgIB?ZiE~Huta{mtnyEcmSHdfuyil2)u$R zh)mGu?CdNMSfTzDEnVtX^9m2v3A60A+Eo>KR^qnE@?5f)nK`F0P@Hm9bWyIemFaoe zcfyY`Z2LwA8DZ|zC+mg+w2;-jKo+Vw9I0xdb)_6tx}2Yz3YC`P1E`yg?FIzLgpQd; zh;mNHM(CJ%a)QnR<~k-8uP!eK$PmWtP%3u+5q;Gn}7nHaK zl4!>~=DyfCC0C^|J1A^ajxNxZ(qq`=T8$nR&{MZ6Kk@~F7CdjZ-F@!qob96SQ^)1u zgeO%3iDdvMvj>w23hSMw*Lo{)mL=o3E{rDfT%Cj?HTU5mVEk&n$Q+M;8+O<&BT1iG z#=`Kn*0HYuwAdk3^qGl>)}X7wBak+cPe{IT>Y##bVhX#_Oe=?RO}qlKU`(V#%lh}X z{>hz1iU~qxY3W)rjYQFWa8Oh7ICqmHof?Y`p226Sx?T^nG}?$z##m0*^vvz1=U6Hm zvs4+$go*Tt;n-&aD@S^b0s;H=;h>LhEqq!T02B#^Dk5 z;Y|`~?Ra}r9#d534UZl@3d$TRRxx3gxCfQ~L3D408tKLj2pHhfMpY=AAPx68`=sZz zuNJ9m$2qGPQ+osT0A5|Xl;v``u>x)o`#Fa&sWhuGb5m1#Irj_Ru)pZXD#mF2rwdFdXP|iitj^no7@*~&6KlH!2yNtnnPP2Y~ zIF$x$qa&jjwWAU>qh68TlksRcfz(^K`d`rV9Ks~X`@3VxjpTIn zf5-Rf6E`ox*Go+iMTwFk9fyof{iJsmVL=WX@hR!)d4~(rh1>XF89rQ`AI593bOi;m zr>q?b^H#*SR4{quPD{bm1Lj2xDWlKB7ne1qM_aAG$(sGpcv3F4&)7`scyMc}DFRxY zBgW6HB*BErzS3 z%g1d9zo2m2`r$iRr>VjmLIAZeb{)22fzDEX439UbHOSP@v?ww=)Z>HAVG0VFrdM=i zn2c6(hiKc)`8>FlGA$V;)24)JLS~d1$+X(lOwscES%sxWrJmr)58q`Dvte1S9s>)v zFU@&Tq#YQRQ?fz(hF$LS^qr6yOTX@>5l>MLbjMFFrDvX7~p3tiqdNq?0+(+N6vYQ31PU1L%J{_K$EVh$&=N#?XM7|PnIi3+H}Jf2KH7B^^l~1 z{-u9fCy}{U?!aQ5*mC>rGhWC@p?=qaaeFWs^f5CcO^oJFsAo@~uv{fC0u^eR!~J3T zwuyc%VT_!u7|$;qlK!ddes2VDP%0b_ifmOOkhUrcpGPV%IIh;R}0m-QQ{9#dL;cRuCjlAxYE zTsHlB7yIK_loIU{gXS$jaF5!i?85zshjJaFP38eTHMMv>3mt~vv8uzzlNGm|jf5n{ zED=Le1Pd`sz=Vs8f{Qx#3$s2Ok!@$yfDam)0PILnIZ>e`P0_jUCWoEZH4wu;(9`Wx zHJ;AD0^rs2UOEIFLDo5C)|%-8Nr7~KV&BPCAW{2opo0~X7DZHNy!v4<-AhD+{<{v`5la?v&v^|OdOucy!Mgpu!XDpZx&&rO zWc$f^>jgts&27V&a)k zp~!=hen(&01|158F07z*d{@-M?VN0Rq;cpb%+H`Q91WP3bo493=W92|pCo_3^;Lzm zy1H70cY5aznLG5m4mb0i-`BOF1US)04~P_Ymo8q_VaN3LFlN!T2SSenq7+3$Jhur? z{1%zKhfXeT++4nAK>YiCcci@lFAK29m?M^g!+k{D(IrKw`S|EZ6dv#ilF*Mhx{~uV z0wUNI&62t;G~1rM_DcpXtvXV1F{^3`Wpt69J&XECP{2RvQ3d*)Dhkzw&LK&%mS+Ml z>u%EZ(S%Hxg7hd^Jzra8$aIKlqvU1UGgCLo_6h!s)*CPX;rhNsK;pL2TUxuxeG!&7 ze}jh@8__HE!OYs)lY;|nyCwMz)ReL?*M|$e3rFZr!mX3THm8_wWqR9silpF6O3PDf za92KYoZ@-;yGY!kK;Ee*P2#!;4?L#qab=`EXVc8wq%k_#BT_L%J^B}5@Zw=;M~or> zrRIUQD?UEnDLz`Yb8_|-wQmb1GJ3h^{iOPL?;cTYf0n#`I|bkx+;a~BZyrPBSZynF ziy`HBWw8m0s&Xt0VgSV{66EYMH#qqCVqf||fmHh+Tnd#!@x_Jo0BRCtRhh*ub&cI} z-NVrCENQjeEQH90>DBDF8*Z`x_?;XsM&6H0{(AO$|g<>(aS1Cmv^T`d1l;d-qo9pZKShX_4{)f&>^*BF4N3QI}lz@Q; z4GqmoYrE_GCkeV)Q)DL1x{ttY(Lg5z?e|)}2oQ1AVbrbLDy&hdHAI7b3@aSFDl=@o zII#Pvsi>7*UE_~)1Y)9JLdD;=~-)9~3GJ^oC@v z1A6z-p26mJJ_HO_hEW*H( z@-vWHX7n4h6Fc_jiDO_(Vx^2WUOcgC+^(^%4wRr;Kqwyu4syg?=Ex{R`JhF?#eOpd znAFQ?JFLy`@G@&@lsE3L4)=fWKkU3qtnum}c9!7-+?JewY7km6(kNY{_}&=E{=ZTP2iayrYdyKj7Pufwc<1m*k#`q_{XzVzaJw*Hf$3n6n%*>g|I= z_LW>VEnBA2lH(wIdvTH#=Xa313T1cLyk7Ud51R=mHxWzujjI3WCA6kj-^xJeD#Sog z6Ddk$XmDgIzh858=)1jPf&+rLe*B@U{?!Jy?Kh-cDO6A>mPOs%d zIo|qSZg2~Q+u{ks#o0T6(x^RyFNbFiR>;M2s+>FTZ=@xHITFp#lZ*6@4#|_n)xw{Z z)wZ(CP~`<_(+M9SC58h<-%si1gzMjn6K)=C=AK{?KJG{tz8-|VzR=;iQ^^{MqafeB z+l$epd7bovTl6q=bKz*dE*&!oI^cX#oSdG-bu)s#r+xhOsA7zf0?grXJovs~4qIrQ zTix>aY<9bdCGI?J8F1qy19YI=g=pqfAOvx!avEV}5XycWwNHh^oPE24NNGSvS7*rf zDp&qUg-GS5UH1l2kn2^twGu!j0WJR-)1|=&>uN3lR@qzNT`xyO9)Q{->lXG?*mp(u zyJxIIst??mtXF;mj#vE+&uo!~n<*{!Q!a=1V>#zcz}2|QWlk|| zvjF7>wRiLeHB~l5+x<1D&>M!#2ES2fJ6MQObX9rMP%>crYx}vnLy3_Q znEqk6H6uDFWcZMgQFVXwdv+;o=VeIH-A27 z$jf^jBNT%R822KRgl5vcOhu_sk%vaF*Y)%pShn zJe(fd6%!=OfOdIg(j*e*FGAO*-e!hW9JE<)2SC@L;kd(kcAY+oMe$f>#M0>wu(0cT z{I9L(>UPFLBqEoNRgYz1Da=XrFxuCAbRvI8w~;^;0wMJ(>6*GNbI7Q05pJg3bQsie zdAw(m7`^aMo_}@KAt3LRlj<9vX)Oz)scI7MN@3_gyz;saj2bq;-MH5`RXv0bWIgG1169>e*upDf zT)XBiD;6Qg$BU_&po5JmIl$8!e2|XM%Nv;vxDjT$+nCow+HJ4~D@_fP^!U<9N(1St z`k<33BK2+gAYXE|HYzVm)>|3R)dsEO^EqW#*#fu5v**)|k(^g8rGxcx8RfgFMde`L zO8@*lz4Sie(?B38jY@+b9v)_zPz%azHtce>N;wd7YiMYI^ryF|aX#Ws4gTH8mri<2 zS=xnURMeE;MoUfDNmIcL=fOZ>_ABEVfZ4O>wIltrM|jQBg%*m^qA>%7qI6`gBFKB# zkmJfkB93Jq0TCtF03%9P_o;R)`L{wGxg^$bU=aF>bWZ97_FP_ufGIUPv(9j04boLVxW=15`=Rl~~W94HawdEF-xQ+`N2I$^4F(+@RhwLKQA( zn3!UP|4Vk zz9;50=h@$Y`|2g~wMz2a4zmoVLsaJKxNhU)i^-924;WiGCmwZEqGzcEBzic6$(=*P zU9m4f;jEgP{)4)!s_Lz?`H>=@!(P@{d;IjG(qoT}Nb01>Ot)i{^P?6V(}v`WTY2}N z>ee_c#VM{Io!=D;An2V-CU4VY&MLgu`>O!-`oeJab-fH%cEM9L?TQiKT}rH!DzT`= z9kLo9mgCy59A3_5_1SS&#w%0`&xp4u3!r>sn9ZbF7Wz&F1J>T|E2DnY55uPcOpl@T z)m#>Zlo5S5&B&v0%xo~cEjBMY#!JVc(5%vuzH7N$BZGKE;q*kC(`{wmdX6pTGl#Wz z>|vp*pY#^0I33Ez>-?lUZiq}j4R1Yl%E5EHO-eBkwWOL1ED-G&2DUU?$H#qR%O3~& zF`VwsB}W%pIq}(Tk+*Cx83I#1X^h84_Sgd~8?u4nu#7JX|0L`Upy0ahzst-1K^>w=M)Stiqb{iHKkh8hZgu z^3hDIGUZ5S#m%B&6d@ezZ`pLArl1-4lBduA?s>Ls%)rWoVL}6n^|IsW#m+((fbz<9 z7UJ%*=+(W}DM)%fnIY%1>v~abk`44uPA)ER;0^H_;E4m|9%p@T-N{I^BANy+JO=bD z_wLu!)~d8T3zE$7ny9f4pw!Mb8?n6dK*`YxvO-?4{U$UL#i$uZ|3)pu%+)f?3)-WP zhLLtzcPP|K^HpVamh=Q7x6MNWLV2d78a|A9)LMMAyQ8`l@3)7Rfk z#6BpOcw)vN3D>RNpXoc=U0xZwMJ%w#+B<71s?BxTgQz93v-IMXxjANZcDE*%wrGuf zJjU;8w8P5F73YamMR>$;xCpT0)9iA`qlHC*!5SQ}A2+18XVtZbW|~B_d4*Wcma>3#aMtlk(-HL-@b*6V__vlD&u@roN()bZ7bBA?s}~ zQVa&<&o(^Qv5Fm1zIDcG8z~_sMJjjc}?X?yr-N`EkL5G zf|v=0QzbUU9#Apq9_Zi2w6=9r(QRpIfyJ;_gG_3uE<*GY#_~hTfNOxwvRw9;~)Cv4|2n-BUR^&Rkx38}?~&W#!Al_p~gx+KMvcbv}n3 z6*nphF1wBwa}Lw)hxOY!73Nd22FEu$u3E2o`b&U?T~Ckb0Ver5SG9|^S(N#il9Ha7 zP4Zc&3(9id>h0l_G3W<5%vO6^_ zIz^t+(adt9_Ly@I3w00gm7DkLpkx|qqw$?rzoJ4-DNtIHf3VlGmy~7Hr9McPUXOk> zIzE~kC_vX9?aCd4H#=v<4pdN9==G#I(zLXuhkJQ2JKR)P^kRE`L_Z5|>tOxB z(GjP?Z_meBw=F11lDj}ZbHLo~?1**jtny{uqPHs(5BKklLNOF&+e#Dv#lgM~he zl1N@f-$ffGjdCWPGbRdQ=*sr72nb6{SEQy8&+{pgXGYEYlJ6iXCfW6nk^s=Y`%nZRZx)GErn%Rt*L)mvY=Mj^+WRLCaqfLx%vS&H8o z%oZHV?%<4AG%o@M(Qrd-+Kc8nDvy)Vr4u+~5nPabxhF=)XJlcbx|B4~W@(y%1~^k< z^u)3zbYbn3r~a!Tn}`-^FnIzoVG*T5;{XUkeUe6F34~DQ^*m3bfWCa78Q;vTxQIaoSv`IM=DS>V)Ufd8pg^bBRS1NO|8Yv zu&fBFId%zWn+lKGH>5+bFU(xXcGjU*7?U7(1C)U2496+w03Yo88ByitL()bfL%I4P z>IHgY1pQB9PrzdAKPsM^DtMjr{1XRv1~ zz(e9ihjJaQoWn!zB_%Q$Exb8yc)Q?#nZ1@N;j+JgS!GGN3#_D5$pl@p^D6%m9%i`; zE5-i<)Y5)MLZQAz<#PgB;eJAz%w9B*@gtlIndsi!<)&I2R$nnGHgM+Jz#p}4C$ zr-K9$_OVl*PH3IcbP#ip+DUf8quN{tG6$^J=}JC++$#>JJTNpN_6%F2VIt|gJdVvd z(~H+c6FSFcXK0PKJ1khCCsJAjTpvxQF_>3ksfm0pgr!i~4jANdCS396q)eqxKG#Ds zmR6G)Hck4VTZWBcPMl^@51JnGe;RNAW3m%r7k$oo6x7N7GP zoYXL`pcDPs4k6?&{Rx9`and&Y@XmAokA-O%jmD?wvs5$jzcF8tl-rNbWvUV4E09y? zAH*n;H(pjaca#bEDD5ry$`qB z^4ySdtj+VH#8s;?ahfAk)U>iLYzO(F>SEvDeN{@LaH%$ru!`t*&M*oYsC498fwT`m z1(bAKT7|O?!KL z2O0=2M{CFFl^U08Q57x$Un1qv5~J*Ku}P^w^?W^<=4K|y#P@zu)$!!cdwhI+Io1i3 zRn$Z;7n@5T#DAb3g(SJ26%%``5H^n2bf&tArdN{mC{=%24g)zt@ zol*Nx7)xE2UL|~4l+J7^F{z7+=k3uqQR2fvWM}8uQq?4nLoUnbnQWC>MWaV(;YZFc z-@v5wom~!V^MAHUN@R3ZYSmA;t>V|q{wsq5(0P=UKIG-)wat)6l2Vi=BZ}?x?ktLU z(kLKNJlNEvoKv%4Khdp>O~gU=NgfDg(1lBw#1`I7Vnu z`OUb)nquwqQe&(SK0Y~3krjO$rQI|Z)dC@etnuu&7s;(M(b=u-A>>QNfqHO9R{28B z`dB^J?aQC*j#EmfA%TG$#!q6%#k?i)%ABM7bjQ}ml{oA>FSA*pf&BA+s=5`!U~Lqs z5DwRclWRzMT|Zx(m?ad~BKrUdz8Z{w?h7Mzg~eniVhsw%t9eP{{iC^f(a0y{Zbx2Y z($c6x+w|#$+*7VHg7c&oo6u-ky);JVNBZhk zOyISDC_%Nqaacz+-qC2I7N%^k zq*TeYCSUR?&qLKNdqF-7F;b16{l^P+-N4nZ2)eX33pVC&a=a-_W!k0IRCITD zx8teTNj*ME6|f@I=m~!Kr3L+~5!e4v66f?`>ZHY$PTT&!gv4b?VWYOkvraqntWh8{ z&28hZ>4hHi?28HM+mw=dhLyARs?dhqpfWmQG!lhXDTT2{CK%HdrC50Euc%qDN%W1P zt#lAdRB)lq8ghr?-+sx z8pXzt1pb{XdV-GVmqNtfMa^HNVy{u`z=3sVDa|?Q0`x8bf^$0W1 z+A+r9cM1cB_q7g?6XW(|54c3Ee&4WHSQQfJ6rwi=yjQv2DWp(TM4_B{pR6M4X8%U- z-yma$KH}+_pg=J8HN5AK8~*Ch6z^PhaQzGE3(&4@lYbrdwKva~g0|!F8Pd$wuMavC zb~!}U7w_kMbCiBtnF80DMLAsqWtrj@5<0vw^8ogNtceM7H>kL zMW;uXWC77UOk|APCWlX}P+kub9#*cVW1Vm(9kw1PVKSl=!8mJKI z^5l)6bVs)#N_UEy)I7ZQ=;$4w z+?1oOx&GP@Y7|03@^$rY=&S42`nTeh5Xs5)d95EF1SY;~zWw*%1D`xVa!(fGEg-pe zP92ZDK7TftU0S6P5<*Vh`55uiKh#sd_?6AtbGzF$29|e!y}F|ToX?teKmM-={`$wX z5lCogel}Y(?r7M|4;rWm#l;Z3@=)sP>!DBId?fV?44o@x)g%|e`L&M8x6qFWnq3y7 zejV7~{|?jxf_QXNQhs<^8FimgYU6V-a&J+*QEuw$l5sshn}73N)bHc1c9h?rPH_Vf zkl>n%h<-nEwfvy}p(GLF-{5}_Cz(TE$>nImDi|40uIb>dc^)M-_2N$+zDK{7R9p_I ztd(45Cs?k1IiVj!7@@5txUabX{ji~gCvQqg+EemhCvxq}{&kUfk)i<>^6h&Gf1KT~ z=l?#|l1HJ1o!h`1R))`RIfUwW!J5HCLH^vCT$u^OpC8YD1cHf%dNtfo~PzH5apw-gH zMi15)^8)LMd-|PbR>5T^-7vRAFBwD0|M7M3_#z$Wkm@^-I#^yRP+O4Gj9?Kn5`P?s z3LkL&^y#*ZQmunBu+Jyj-G1#-w`=h8yaoEC)nB%_pr&-RVcKZ~7Y!Xd>iqN| z^Se8>2FGc(?MQ*m?9+JJ3+J%zSk^h&L}m$t5Z5I^lD8+oW7O!NvXqjUS?}Wb)Cw}3 z7jt$#qR&el(EGmvz!5UK4z4tLbMXCf!-4lIUbI-U%N1)JSfO9Zge{Te7R+%Mj<)d+ z9y2*zjsL0DIjtL`^8sA=KrE5_et{81Mepbi5@3-I>-^t;EV-WM3mkt>Guct)C94J0 z;;@Ja%JY+hDA4zJIlCkyeQ=U`J$C$YWMI9LqGr2tmg|)G$Rir~&-tqsA9Sv~oos{| zZoXa}&8>7BAwMsudQw+cM+q=SJLa^4N#=vK+A~9^2K!a_ZF9I#LakloC#yy?X|Suu z-JE9R(+IDHz#8xAIg4|A+~0Z>;uqDw3VK1cGX4t>4kv(h>~==k<~JoWmousAwVewG zkF;k&YYvC5GIZDi)dS!5TxPRP%txDE;t0H69LZT+9^M_8atPEamq+>T{Z8h)h4cD` zLl;vejG?fRj;~#1WjJG9{D2T^41&7?)zA;mOmb;{3yC(gkS)cIIV&*Mm|sd_J7UK6Fju|a$Az5;Vrr zm!Q#|c`-1m4d%^g0i3}j>$&+wZOc!&H!(w@G6dtr&N>6|hQ7~SJq>0Era!`rY^I!+ zoO-X^6P;$4P0|(m$V!a52x`xRABd&b(XBg573y*AN^_f5*W6`@atXF2;P`Ccs-^++ zx^cR2Y?4^KE6yvL?7TE`&+Yi_X~)iHG_P;^+mRR1y^sYCCSD%buQztS8iaI2<5NqX zpX@IX;~)94L(lEvDiCj=j#;#y>eJ1&yIEE)hxn3tOvHC-lO0dLOSI_qj$P)FhZ|u% z6FeQnAGq{qSQj*Z-q>p}vu8VHPnNqNoEPQkiC_|b->U!12b*?_%C{B8ze4FBm(gE; zqG(g-8$B_4mc4MG=eA9xS^Y}WU&FZFu*_;D0_G6iTRmR};K1+I(RyzWAaPhj89m~? z%>A;ISY_P_?BkZFhHFb)2<47_WYrE$Ljw)#A5szNv50%(*;(XnDgnEOg?`^lngo6w zN`rRy6`~Q%x92^T=3B;x1(O*HE`I)e^g%0eyt&V4`ZPIL-r>xj@s-b~T)H&zQnuFK zOxlCmN??&YCzY9xJx}yFAad#HYdk8!s&eDBP8H0^N0aBx-!zII7x%b)HIOc)Qma${ z%%0AYEs`yEWEq9MKrbLsMuup`=3Px@Np5^GyK_?M3ok4Yrq9BCHIt9Gi2eASRcrKZ z{3c#5PX4buPUHtJ&ypxDQ@-PQvUa-jz80C1?#TLbxms7W-)W>?QeMReU*E1DV!mt!NbkH2;TghfXV1nfEW%3`I;DY%+`_Lu?7@Q)lS4AUfPhl5%GGbpH*-$( z##Dw2G%XI+qi(nObn3@lmLU?D0NY5ujrvvjeu{6wmkyG9#Rk2`rv*lv1r3YDqYZAg zIup=$U)#QZUw1AzIy#zPc>k7Ut6$k`x!kO-5RMeUc4A*Uc48p_l`Y8J&bbH$V9OIV#7j< ztjP5kJtJ^dp2((seR$oMTpi|_8&>+Qpy7nr(ecmx;NLrFG6F_CoFd|8ENGJL@L{Owwx#?(L z=Ac&^ok~N61h?7LmxjwTc*yMPDnm~!cV&!1EaLu9x|9tgeP^!~6tLLWjZfuckDzXS zUF4H4l+m1nF9yprO9cy~_1q>CGmO{QOjH*Z>{$u^on-uxC(hS5@y6kIqKPasLHq=n zDwoQn@Y`Va?{2y*dJJ@D)>*>kj_VTZnIos%PNYFoWqnJfK^}X|b-VR#^=ZRty25mq zx#u%ScTPH9S`|35YZ*jcX{7=T7^%&cjs%JXqlo7D4ZEe#+67>f>S)z@P~x#YOD)5| zYu4%Ztw6VS%~UcQ_w=HNTg7~|*4=o6`tsFT>SyK3ZyaX0Y!ed45^@4MSBwXL~(xf99 zpw-^gJ+Y$}!skZ@v2M#2-<&cm$?Vcxg??ZhPq&10_}H%+?yry!ti~=^wD^8W2X^0h zq&i=0ahyfsxGdJK$15$%H>a6CTWwa0`O4!F6286xLB(E+rVDS3lGm;2Sa+N->Ow1c zeo?=kG$IUy+)UiqHBtmDT9lR3Y&?a&CIWxwEFDji1L?#_&Ak-?@(}&`^G{jl5)4FS z4n=K{zNviS@4l!`r<<^7J8y|R5vJG)pQ6*Fa*!pwVxk$v6P&h8twFIKQcPHfzxS%-w=Uq!E%*o#N3ZEo?euTv%p;`=bnym z={{4DcUkkxT$>#u9~pD(=@_WaG#x51TKb~(+FN;SG3GeQZo2*ur*w5%R=$CzHq2KT zH22IUR=*X8tLCX{9;{CqjMyh;DH#YwUUuu~@Kq9E{hfgBs3#jr(xIVahCPyaxa;uZ znOJft`s?I_qwVK(x;26EJY*(Fm2^w?OzD+97S~f6C+0W3i|+^4Xs{kb-~%p0xirz$ z@cmWP$yZB$1LxVL>2@A^oF>>b_pQEE^(kYX4t@1Y+y^GY&QbP1{MyOOR|`W(AxoSm z=4Z&7)r%4&MJMjd%A*xbyt}lmqA_`|aaVlRa?N?`Mt2tjDjj>tx(q^u%$LGz${fix z&(6___F;sYr@Jg5wY-nUPe{GlxLdKb79i&-yYyjaXD6(%$5nBRqo|?>*%Ju~$^5bD zcIu|U`L2y<*E+;grt5-9w^+H&gIDTV+zfn&d{{DofQeGzp#Y0Q`}4nd**D6`U?wM? z+yQ90XEq9&qWH=cO^Dgg> zSK}K|(^LVi5|7JBJ;9pM68_$8{Kohf_x7ju$Py~vObOIh*{^bue6*=B*-M<_zB`cTtYW%H(gHP|nFLlFf zZ}XsgrP3vjDz?fIJot4r{k?fmI3npiNLhe~yqUMGD|H`i2@y8vJ6;KZZJL}SJ&|>a z3st5vYWLvpJ;;U&CV`)Ljm+S)YA4&8(_MHQFu?RnD%1~VZ{L9nzG1pW5!#3xW^f=C z_QLxdD~=<2UzjHPQIEa`Y>48`0gCCcfWpI&r}5|bw$G5!y*!QaET<0&6Yg=bSY;|N zK%X(yxR3Z|O6|<*oT# zia}7+Jk`%N)V#PoEB)^s{QrK74dYD0+#{a9_X}@|P*q&!Ftevue|KymP3fIQ(lrV{ zD!}>or|uLYc%*eU_W3aUj}#se89fgm7f7c+tKR%N;+bv?btZSF19Ex zmaK?TC0+CEJ$aO8lL#7!Y05bGVP}%awQz07VR zH6Fmsx4wFxbrum!!5)Exq}>?I*H9>^-{7Mo!<`{2R9@&Tff7f5{pAJVydL6EYci*) zS9Re-G9M=6Z0s})EKOkFUgAiC^rdtyu0~8mBzeMSJb`UK@pPTA+%&8XLENi6N?!f_YdDjZKX+2iW`8&b{9Ft3V}{l4g% zSj0RnqlY3Y&iT5v2g)f=Q2Gv>_g7?Ee6ZFX?yuTufkD3u(McXc2@eF?50>E?c=-56 z>#>}P7Me`yjvi2%=!560+h7kbXVEQCO7r>23Z(btT{S(EgC%m~6{&s9_VA@BKAEm% zc%3){pk%~-{rc5EsW53$2V_G1GhsI){Nt)WpPipW51j1RjuYnM4=e7KN4NN5!%jDY zBNn=%I$H(9-su6Ze#A8Q1@&e-lwobLbYOY?3dxHSUBu4DHVQf-$ zv~iENR9pa`WG=>7EUU9LR9bDV;r&-++ro=k{k$sOCDdQH#(%0pkv61mw8&yCmtQx& z*F!4Y&*C=+3o0)^<(*VDbG+UgtwN*TntLQAB}M&i{xfL+pK&dqGf8Che1^Qkta&+E zV{bFrWR?y%LOUC$n=((Irbp06lCEi%+eFf>4^NS0wpW^ssE5hU7)^P9pne=iO2}h9 z`(DRwA#!I#-($=@+e5%adKhw``c=4eR6p`+FZleL{Ay7`@@ted}izDBNkPohWj+41N~l3FI< z9dx*~*)kA?N(vi$YzSDJ( z(!EW`7~PB4d+h@~5%&f&`i`LY^dq+Cgeq6-opo1WZD8tOc<}o8sAtS1#C;8*duIo$ z&eg^<`@PiFD=S`MpF?T;S^F1{qoaUru3xZ_G199A!##;j*!EnzpqqzIt=6oplYoWw zw)NtfpM%rR2>yDlF-s(qMz3qsm-UlK4gE(CsAT+Zeku#PJX1N{I8M=2q8sYA`2fD2Brty`9BXyN#3dqdgC!S{<0!4q;G_&Pko@vS z{|l|Y#QL>TLP67rve(0^To(s%nB+naTNN#sUo*`KT}~LQC72+EY&iT>{EQ{dizsX| zohptoAF!PisI|EUTVB91|ic#m-5Hfyu zo?cqiVE%NIYjDn3bS-(;wZMmf1VchjIimchu6&Nj)V^ftE4wU+^^_MeE57%JT3dGy zyWkwc}4~gojbF!6Nly<81*r-FSrM08il~|%r_ek3()LPy7q`l=S#XyN7uD-mk_*=$bxRHN1TH z^hQBLXSftJ0k~egDhYi!U+*|w+#bPd%V3lVP}8!>**SdKWHw$~`5VepdXUSNROqQ? z-7*0PEY-(>=KGkJxv(PV!*%X?7hA0JLedwNeHRWsU^>>|%}lK_^Y!IbO-WAj2$Nc~ zW?035=VEadZEa>M-F9XnLH;;ypp9nn@15qawa-FWI65Nz_t${M$LmGiV?z(mO#R7) zCC@uPsT#`^sR`dPkJX*$IG>%G#3tpTwKeuI2tqiJ$WL)>;e zft5E8Ut>6YPrzBo!k|6)(~9vFc_ySSu!3?|*byG)J{CV`=X5aPl!}--I_=IN5lmLR ztGIknXS$R$-E^~Y=T)7M+v(U#P?$!n|6r>Xwn|dzYd@c`d&3l9GA;M57ZQ6vk}p6i z#1_e1ez}ftgtfQWX?tTyL-rE*?Fs#n;-eZCHh%1P$f^Z7C`zF1oEMKa7XmceW!?-O94e8#g{jux>Ewr;%K zn+tIe%F0E>3^fMh?KgoU&^Y4Y*tqjfTtXTVpL|sC>G!}W&MfFIMLzeg3gkduQu1SF zE`G<#M0G%fK(*VN$=v*oJ8V>$k9*+%Q1;$YO*h;3Fd(9+fP#o1O{CX{DjgJ%-kWq4 z2!tZidlMB=r1vfmnh4URW1$2{0)$>wN~9w#w7@&~Tv6}+uHRa({}_WL^O>15bIv~d z>>~`o)x*^$Y}180=sV0^n_H1+m&Y%u_&h|sCw!h#_WVIX!1zMQFCT2Szr%D6govY#V2g! zfV1w9lAB$yFWHKlM~h`V8+UCU_t`fqcVBGIi_7G~i}xdcr8l}gl$mOcMbg2!=JJuM zRrhfy@<12TB61eFndk6zmlUG3!J~#jqRecyHJwx1G)e!3Ybuz1>6c;PSs)*m`Q-;_ zeLgKzkUNVlmspL3C|eJ%ya(NYB(00Wfr5d_AuqTW$zx+GB8jGil=9Rwue;76YUCWI zn#>@GwCi+W-T1Bc@WbT2*TdRjponn&n=0Sfq4;*zt(&=`gN-o|T0?_)G~Z!;Z)8422q$|eG*s$uQGm4np2=UueJ z_>Rd7$GUG;UL+0M#=VK?g`u~xUvWSfLghH)!zh-;goX=P zNEy7p2X4Rn{5iIlwqj1YjT((R9LVL9e3K*XhpLdAngRQ=Ig*$>q?$n)FQ?f=F6`XP|A;^G3v8Zz`@=IgM6oAMRqEeh814J&Xh?gs$tc zr{gEDYHY_!2-Uw` zrcL|TH7fg>@{vGV-+B3YNv-_hB_@Be>Du-k7hs6dtdBiGvDUA2hmL?nJNa-XC8p|D0C}E{QLUwrF&nGCB zBYyUS+kd5J!$6dKd!4bC_)$ah-k;w)hf!NTdW*A=O|afw6PNo@9wwN z&zo#*R2aqRR07-D+ERAgJZ9>0GayW&!38<2g28=$HVeu8yuWPu*r%US#9uw|(l(_y zc~kcAQqLC({DOlG$@ei8To!;WGH z$?pr1F-SJx%A+|5F|T$XL}3qCDt{xi(Zl09!L5=ZoU zGoB%}G_v$b952tUTOYnHPscLwMcdzIv^dMm!m8}%7AJq_PDj{u`$ns<^QDtr7;yab z44LPKo}6r57#{qBcOm?)f<}w#y7=@h%rlqATE;0(T^5S#Pwn^v0}JmKz5seVN_zdh zlFh4r&atM11m5fyFJ5HVe=`t;^Tl?QS!#QW3do(nr%x3bo$OUF(^IVwo;-{YK?*|F z=yvSpe$7s>AIrsnop*$%i7;>Mzdg9fewLsW zcWRlc+uQ%Gb_t{2qp%onDZi4o^JmYJ`?t2W%`AA4-p(pG2co%=FOH8Mg=MR>_@Cfc z$$*^G85f#>C&M178VwOw8Bx=o-^AK-67~w zcW~wTI+hR8xz?H)?Q5+?c&=X7)Guv^ii~}!297(KrdslQ8U?0h&KG}eEUkBh+e?!i zAZU+{>VvgvTxylnKD~(VQe~shcwJhX=?0mqmHcOFnNp&h^-m32#^r+WgGg`))m~B} zcj7yYR(v=kO7x!@Ki3S*_~TJ{t=um&ZaMcPl!4rT_dBgN{58APMOnu9AfgB4;{?r9 z+Tw=)g*Afh1q|$y?hO5(AC%Q{{`kn5l$+QiLE15Pj*XcdygYMuqm^|C~q zB8%3d$(PymWC4$VdRYM<@njT0vb~IPP20&?{mI-2!{Vf7L<86pyg-XL@kaEe{NY9o z9!S+?Exx-1i$uP43Ql1$NzFNui`;*n8M3{U0B>^3l=A$AQ2(6RU9}csd3^!vUE~a{ z_JTq=wawzpU;e5{AOEu66E%PTX-VKzQzE>em%+x{A~W(DA82TY zD7SbhuJ_hhA5iFpi7#B}Mtr3W`E?pG1_fPw@=flKSq49uk;CU{#Q;*7z*YS+9epu! z=b}yXx1;X4`z(zx`x@0;vwfsaokA#}J|kX8*lS#PLXu z6qZMikw)_L3a*^I53dK`@x%z%}|`U`{K()LzhYpcG0`W z)pc3~1R9X0+*smx{d*jDFi!;d9F2?kBu2C}{$Xhh3V}Q{vwh4BoFET1oHoD(K{~54 zj#FP{w9?W3lrT+kZvZ7%9*Li@-S}{l{x)sq$m4xuA&LSkrRU4$Id@1>FXfKW2-$w2 z&i0ZG`YxS+`r0Q>{UY%<(8-sST&%+O1LZ^G4GQN`TMO=sTZ=e5->m_|T&Go4^=jj6 zOn~sazDlSdDv5@60bQTyxB2x<%!ISl85_&jX8YR^^Kz;A?fyl5!$$K)2B~J_!U}1{ zMk-IDcxgUt1&2v>@prDqJXD=-KCqh2f-C-NBb&+JrP*u^OU?a# z{pvlIrBFTo@{y{%@lLFoG1y6J4gg#A?LD56%`o>x7+1kech^@|xKEDlvoku3o=3Ut ziEPvV{CRxHg7#zQZU0fRpz-S~W+#!=kK@b;u6$zQk&>8L!~D73x6VNDR>7w6?m5cY z=4BI2U;;r4dP1BPSuuiCalR$&W|Lc3nfdg#hu5UP;n(hG_%6A)xrKYx3IqyKVCDgv z=2khgF8x{#nM|DLWPoZ@vbY%ls+ieD#eA(-CmVIfo8QNg@;J78A2oW;0u&+s7~SR2 z&G&hH)mKdxHpC)z+;Vx|94L&*Fx5;1)MJcaP+=3rDr~xrq;`V1O)Kwy{Pr(oJPSRF@psO=aJ+K3-N|i4fL3y$bg42k>4I#+q^Lw`(pSBpa z7I=8yQUOn(V>PvoaXH@@;&ZG}(zHofW z1=nPuAF7%Luh15%)$Xf<`YVl?hu1c}e#{!CnTZ8h9O49^6Xj1i+^H-pYJF0^41+YX z-6?wjd8-O$03L>jpk0TV=cx3}l=7$ZRcR5alUM811yh8S;>6FoBU*w*@L(%GxHDw* zO8ifxo1wbRWvC4lBCxX&OHzg6RLJKx0AVzT(Jv>1s;0Q6_De=o83X&MLJ9Uj?r&XhV5A!UvZEW6%8^IcfwY*L8Lp+vDw9?Rn_fGH6RAfXOWzhmFt z%fMw=^6nO|S-^6JuuL}H;sCFvsXssTP)4@xjB5(KRK5(>wf{8PU-&Z`Q( zN=eyQDxGy3-)!q~Z1lyJ;dim9>Ocsf-t{{>gsIVaI3ZPN1KiFfC4sEJC9)c%GW4Z| zKjak86_4(;v;Gh`LSLb$=cSWiJ`=U{ELKJ+s9-r2iq?F12_%9lZ1duY4DvNIp#q5_ z@EN!Td|b>m*R&S&&P60kp^0et>JQGyu zni?W;6p#93^?dNzQMahM)1Iqlq^byJ>(uPTTAu><1kaJf?$`<`F$c@F!2JPUHfH8r zQD*+@{N7Ufa|w}e8Zv7lYlyQxk9EH(vy|DQJ=$-3RlaX^XT;kb8EX-+JAxFO?dF?t zbF;{=f%o^_i$bd;#XK~Q&=jmp+g`4zP)n|fO@K_JWN_@@l73e|tua|&V*917DZkF6 z{>Ka%wpM4VzTQHk-JN04e8r)>dvi1$^JPim(U%~E0eUxcGhng5aC)hHl}6MprBrj> zkkpwiy?+Y2;!v+CeK?G~0ox~K@Hj1-j%1sv+MG%onHm+wadVs1$%{Bma+|3uH%qKG zsBWeRuNjWS6{iU~KPbx4e7lgQTQlKY1Aj-uW1OKSvl{;8BJ!s6jeQ;kI0t68}c)fp)B! zd$$$7_h_NlW1Co_v?5X&O0OQ-I3&d^VV)$NCa>*L_-H0;P?ixlN^&DD9OxN8OX~B} zxHz595Lo#k83+Wy&qI)~F1}{d%I--_<|~Bld!aGU)lnW$%~uB7S{Qu82J(^7PI+o& z)Ot6j{BEcp-B66^(AAW&FMm*{3RtmyOG9EQ5MW6z=5|{zcmkCMU1%t}_+jKgcy%_9Q#T=`CMF#4w{N)W?Yj}lgJL!DS z>kw0^dqK29A987g?VclNVapSqlN$WbN<=@fFUwf3Te%)+Y>1J>11j$*c4*QI4S7le z@`1JUc^^D9IC(C;_nTEIG^?+sl}I=cTLGwVcZzTqQFmJG_q2P>xVl>_4U6=>S?T5* zaqw8?Ho{z92ebjNz=(NMQ$XJ3nRA$nlfObPd_orvU`>gSX#aBO&t8_t$GJ$aE9EAm z+4Y!i#8hZVCTk)|#L#zO6i@oreZ;BaB)-p>dpzs$OlRY)$h-B9%EIdwz4s=6vVi*n zygQZfnqS{i))|JY*GrqW>xw0f)*l`o?3UXP@m?E#)aH(Io^^1YJVR3VfzSj3pK;0? zRW~kKhO^OodgOaImj}vt&(&faqvjV&S>cGAQ=FtvJtm!0=3ni)6_Xg-LYvzeeET8D zgMCzU-y1`|{XU2Ew93sA3kFSNHbzikm)jY}=e)@3;|}EITNu)8G|! zp$QtLmMZ-z0n#5*?KH((p(guJt{z9NeaFRQVc%*x)ry))FPyN%A&NEYxR%uQRnF|* zVu`wQNjz6Va&2Z)jb;1iw7LY%JAt<4BM0h_7Ub)maS+`<{R;$ zP1ARj)gIda-qrBlt=Md(^8WBOSybZGcazT+~tr6{PvJ-#eIL_4SAS7iV6=Xqu zl*bHmnB^7AGe|%xWXdUmbk4E0%ygfhU9(MyTS12xrd7Us_wGWyJyG9$Wq)wkW?LU) ztLKzITg{YnC=V__Sw2Ns(0YBSy1vAofy4?%)qM`=KspWk)gl1ZwiQ-04ygIPeNX0V zarYm))qAeSC^GnVH}%iNMk zs%WIu8V3SC{QK67x&x3AA!dhvE2auS>YTElt=;|DomJ3el(D*9JD{3K8sGBe9-zQE zq2T4n^YQ2wU5z%GRR6Bz3uSVlA0=7lkaM-?J8S*?{FY1~P@hgc*Xcjg>K;QJPxT@^ z_&;&@E?^lJ9Bj+Kr$2F?kN0ra+4GkZ@ZzLz6L1*?M!9FOrImYRpRhHw`cF5!6vx9x z{to8ZMSs+(a_(z@72Et+^?08ttL#O;51Ql`+wF#UhFtdPNdoX zo`U~#e=wSWRViH%Hv@az^0xP{n|PI8gB~}pzq@9vZN=&1+AiwvmFUM zTpvDsAZM^?GR3NJo=rpuGd&_DZ1{{qDKarJ&1l_C5(~V(WA$gRl$kz)hA<|ObG5wR3C%O*Wl$ zCl?e4;jv%+Kqng*jcQpsz$H5Fei<&yVRe0%`y`H&C7PA1=!-BPv_H32p5bLo((u)= zS797pH7%4>V!=8vIhab7olc&b(MYwsYIJ?z!{3RbaDvIFKl0xHUbSFd0v0)y5x#6q z$NR+igylCbQe5Ntz;e4BEj+bVXxJQ#Ym7WAqlU*hB}K=}eghxZurgL*@*;If3U|Z{ zN-F*XLd$+Q<~y!sUiUmXQ>{3oz@Pa552hg5Jp$q%KXkfLe7N*oKl^)H@4i#*2@#&l z4hrxP=6dnlJ=Iz}KIjf?I_OTo?d0?SpnT_~mlPP!S-Fo~gGAOZzie*QVhWoM}z6FCtp}vo=v+1EYRjj{H^~nc0jte+MG3rnEo~9;Dh6{Bv6X1*?mZLpvYUWIU1PGxbKwKkF70rXOp@~YOCaP#HC&ITXHO1Z_+;rdWQVD1nsF>iRq98d!B1srzH zVp&n;^w^e=Ba9IW@zFXF8|yi>&v-L5W580HdIS0_IPVO-pBAqmerAL1K#^gh?@FVl z9jH%z@IHn`wKMgGqKd6;^5ro>W9UFp73j^C`VA>+*OL|80Q4oO={|b=aUTBWy0Fi|jGmvzO`c`8doaW$n@p4Lc1BxF;u{?w4w^fvYhp>LTDMhhQ$r zZ12Q+j7&{UHQ7VZS*b+60}s{KynC=FBs) z#2AR_gt;VRO9UFGYIw|qL3#I?*iP&b^x`E(YN}uXI7@}#27q{+R59QPZ2P2$II<3z z1%@D&dhuTJo;j1bC#V{UF6KrQcNSQC76a^OxU8~SM0gPZC#IGwwGJcYeeabip#AU5 zHdoM!KzEp(Ec_}jI$x8bF&~hGMlAcwZBXVOBe)#5`92poi_|h}qhfv}jh&VgCW3Aq zkh3*G4DxWlplZ_4*0#X+YOhTW5rE}Y$|j6zD?OdW-W>(8orPsOEl*Y0XzmxM1*AO9 zLm|`oVV09ZI}mM~-)4+QkW75c*U;=NKDGbvUSww{P~JZ3jbXmduT3KYY!s7Sd zoD4@B*IUh58Ulg;XkL|Bamd)-M9m&7cW})DDUDX<_wvfb67MY{8nLC{f* zlak9Y8bGuOAW^BY+!4ak{Gc`0u8M~IQh=yp&fB}YA#t2$DnKBLd+5S@74Qq(s)_!5 zKvMwlKgqgQ%<5-fhEV{r;Xf3%95>Ei;f`{$LB!Qe1}K6)ELK}&ZycLf4T~?6*n$Sg z@XvhBl|~YPl*wWrcwjVGY-Slry*oJ8SA&IR2S<%PY?e#8=@|3PcuIIgG%@L(lGV&5 zVu8%r_d8zD-r_Jx+){ASSB+5v&X=@Q4`VJmYB;jVb4J)3%}a6FmxL5K7kpq1L0>eO&O(@#>o)w?NO z&=APeaAL_CQ4fb?r(JLt;?6~b7U^TI$n={QoQ8!=&d$zru2J`ROe!5XET!qVj&Rsa z5|Ic1DtqIq40XfPskmAXUcQ8Q&aa;3vH4FW%e_Rwo4-JNSrvSo(+eQ~`8*QW?q&Wh zeqmLK!`G%EVLda>dOY$E!oGrR3q*#b^aegPVOAVtzm+^R3M41RDW1?c5Kx@1G7J1f zlCkszh`9CmiTQb!I4M+sieV#j!mLfEJCbWzl@0@(ty_N54$<3Dj75E7hjyA^^Q zn0NIy->Bilz0?;EtbOrP;y%Fz8b1y;7PWth-c`50q#FC8OiW?gE8qiYS2e`apU1|e zyw-xo?Fg8LNN5?4xr4~``VP=D#R~LE5|MRZLUL7#xArrkh&cs`Ij(#~FRWz|v5LfG`m@Td)X9-D zNOK>7=2*VrM4dvmg6pq_5E~rV9mh=X5vY^-lxLJsi)xvJjzG{rZ4=){jWAinVBeIr zKa@4Ty0~p`Ed;9$(Q>NwUSo`(xN2+v2Nvs|-jlD{~f=czO{hUYzRfbsqez?_jI+=iR$X)mE!^!~@?>HrfBP z59$=EHEyUwrShRYrkUyD^MZBQrWx7T3X3WGfnpXwa$I-l+i2MN=n5FvCW*+ta%-+a z{rc*0SM0@&@iSvVJ@JNS)|tMy{T`D@ZZ#jc7`?yOWE0+Csw8lW@ArB4f4D6($+8|H zsx&|*4JysEnme1asrq#@C18G$_qu+!6|ZeFn^cMS%PSdtQB{6}PpQmp1%1b-ZS$Tb zG;|h0skxTl%;t`oyG7^ACnjL}#I(}#FwQD|^CF15E{Pj`j-&Z{a=ppBJf$Mq$+(m`bQh;~q)(4|O0zp_0M-m9@UWbT{qHeW_D38&3hZ z8a~zJ33V0~??H#CFm;l0jIe}_4#l&xjj+77ERLLwrsZJ2pai_${%gIyO;Nsp5X z1X;^v(6;v&E}Cz>Y_{c^7f4imct3t(h0r`rMp5XIkliMmNDjxS)eR^x3HBeXCY($H zeGP2GE$Z@KE^R4-laZ|FRY86PH@7R*4C;0qy>IdG7X*sfb~$iQl;RhQxnjDK$};DR z-~?QdDcz&tDtns+bq}*uDRxn%ez{Yi0^Oc+TXh)xAMw)zfyYvGwzr6?>RxkP7SmG#L|% zfLUShqfaxkDS`Zf-q`dSYuvM~b)>{9Jlv=PPzp$?Nt}(xg!g$E8NXMyeYtnyy9B!e zaAILx=2Xc^MA<<_kR1O7eZF77Z_mZp0H2G5#__GHXiV!Yk~is-7eyG}sL`B<~6Yc)ai*F)Org!aJNz)E%K4k>YI( z5XXckH16NmU#Hrh-*R?B*V>63RQK*J;7a)iY1kL0j?Njqw=GMrSKObHad^*dUe`~* z_o^1ts+ur@DL^?)>{&k2F7m2jlr{CoHCWcyLSV*QG^|=$qj7#~0>SW6*1JY+{ReiQa zEXp(2!h1oLlAw&{&9JaAN*N0o&8g13gY<|u_z`Yn@C)oD;%1M&D(K`u919;j#sht* zcTVbGv>&;pC8AAzPUF&Q=UfieE=0z4>f88!qUAP(4aD#QgaWGsb|t)RZn53 z8?#_uZNOD?OYKs-Gi3T0MWWTXNkr!>hPdDOv-ae@otpJ`^oWBf)e0u)M1Cz^lvLfr z4Dqm@UwF1yTG8wac-x)?H7kxQ)+EDr2{8$h4exzD3r$Kh)Nqg&7#iM{!B&qzHo@I&bCaUI~3T)IFMtwz~=Rm43yCr)3#Kn*Aw* zxG?0u!y#mftj~t5wL!Ti-_l#p$tvp8RG!OBpSo$;fi?_UC`7hPy~?iF^x33uDaU@l zj!^vv=&vdsBLaSrmw6Z#scuuj$kC(cNpgF1ct|{bu&U@BvoC^(|z&{G1`i3TVC(D zX_45aR(O--kvO$h-E<*a=DQCZK+2{&lg{J_*S@XkFFlcDyV45POh(O98k)1D9v*Z- zs7R#5$YLs{a2X?ajbk1SAP{(^)Ri|oOg0!v<2D5`YnN9m)SiJ*7*1r;IEuu20F?}grd@R+nw%nb=DuK!{_eTi zx3LgaF~{L|ueYTK<{Pmx?<__4H_a-8=ehYQA+SJWIYM&g#Ea zb@1>p1ZZtkvs}imPVvb-8|P_G4Yca1G;*bu+PeYmFNh{KC99xL_@pSws42Z0N4MJB zPA~YO`C4BZo5ZzE*W%<*aVirW(JM8)+Jq-5E=eN+PZNZXY_I=g(fA@+P2j{i(dsAk zdpSzt+WgyQ=Fh+HS`?$!H12D z`nD03^BG?^&m)g(3byz&{g_f3vu2Ao&wpPYt4@~f3h;(^&mW^i(8-3{^bp9a-Ny!F zlCgA;1_lhrtkdmbpm@~7D9}yVHPsd^JMi%-9F{NC^#GAFAd>X1)9NC0giA)3Y005- zvN2XL5vVMrvE`%5&A1G>QQLQEl(aIPSzY09^TK;77wAPFF{1fFkP2xI8ua!PGRcNk zK&qKjA)Jl7bJqsfs|+nJ`HQJ{)!_ol^^zb6AkByzv7XAwi4}ypUGUsrL?Kv}ZffqC zPfjIVGWo|Tf`oBECRR7k53zjj9h+Wz?(P~VCJO4 z$r^a5bHS?gIvipSWi{|oZ@v)u{A|sKr=9vmVNKZ)^!%kctg`R88s(61k!I6?-ibm> zU(}Y!Lod#b{&B*q=_0_m9p};x1H>H%I^+A+4)#Ps?u0Od7C_D(a5jd>YQBk{f?NJ5 zkNil8apP{=V#YG^}$ZRmk>yG-VbYzu%rU+H_w%b7MJ6ai%7sAl~e-Qq3lI$Xu>Rns&?hC@T zCZLgyx>#e)MCwxc;lo84(w?jaB8DTI@&bDQo$LEHPp9*!by7~x1`zqYGY_RN-FONO zGB&ATeYGr7AM=S}&-bb~^X-qMD>tWilTl_-ac7Z-ehzzFbhM^Sp*B(#6OC&3v@=`3 z%AG;Td{#Q#SnbY;?)`PF11~7*z~UL^Q2af?AFdJi#SPCm(Ghi;sL`!K8S+LJ6*pgv zQ(8V;De7O4yeeh{%Pqkn->i&n!%6_nS!oTY7$j`ksSrR>rxB03)s(kc8{ggw(zreRrZ`X;`p~FK3Y$WF1|_aT_{a2MGRI zQ9-+v{hlezW}7DN!Z@mTSFoj#C&$&g64wEr;-6q(V6b@%r9ML@NOMEx;ZWcj$Pcsh z;$?4OlmZ$wN&s5?FGAG8h2QyOHlBI<_uXzPdrCtCeNS)TlDhzjZ0q|yiT2v5Dl6U0 zCP-5}DcuafdZ^g;mG_s%%XA-iOUeVTQ`M1OlW!!w&fC$#B^IpcJkODp07H>pA(J85 zRYru`orr|3(MJ24fl|w8&1?v~ly>c}NCK z%eUchcpW?CvHMyPlswv-`9PgeP2pDt-|n{((bEU6D@)kCN}xfnLnBo0w9gJ~gG%bu z>qGCIbzKj|MNDpEB2`ME+vr@r_OAAv$(tK%{hAb3XuLRd^}~s8+&+`oZ<+G)T@Wt36|p!lKCT1JY}&dLz~u*Y3aWIBy$N2(TQHvoV?c zl|okc9h_OS7%1KKIulE6Pkn#-gZ_8kriDkAHO?f&`j=&b%&l!(EK!2wh!*Qbee9<> z>KC?0-J^6hQ>?N1YVl>6{tpF=yPI>#BGEqRx>KD(d&1L<1l99Y+BG5k+tNiQi`*wk^Txr6K+t+dlAS?d`zTpBzh)M zA&0SW9*yI$=%VwSWT|LMzhMknTo1s3B&46Y*P+;aU@N6gr(E34e~KFEsb*!U`_=M|Vk zmu0A{CUQsj=G+NGqRZ{JXeg9MR!3?AaN>ezy8lBvQ4`LGZM*LD+k9TT?~#$}eg4WY zcXlbOddx8?&)SnU*)?1+Z+}86z{hpo*5Xa3U6f3~6#HJE!2lt@J2F3>tEpyj?~~L+ zz!L_=kl55LqsJiZaoPhRJTvRnsxd1a=x*EkEZ6hW%uJ4SN>&LEhoS50-tT28TGnrl zJjgE>NL$KR$(zX(A-QupRUn2Y_h(VR5d^tf!TgyMJzXK@Tp)?jZ9I|vWOJSh_|r7=;~yN^-d9~Ci1hh zlV;;mH$}XUug{{m({tqgZ=~NvaY8xF{IM}68qML8RaR*U>gaW6?}18(EM#=$Q1^XM zJse{|W7W$?A!s~URwfwdrIPBaX6Ctyc6YQNJY)_` z*^PZj@29)|%;B)V%)KP3f3|Y-(L{;$ngC}=LN5nE<()b&JcI4035w4-IC-jwio-tk zWlSlDsvMOni7ToXB*!|mZ@=|;SWwx{=G1#Hjwrj*S}z+lpA%fvzze6eQsI1-z~)|=VyH&j~9Hvl;vvLnxr=IDKr)%($%%t7p{hLeJbl^UL^1zJv0T)lghk9^EXGv zd^N3CI-YR)WYVpi^WGhmIS}u>~KX~TvLLSt2tM9y5bj-C(5$xc5g1U)76~l??+|+k{Vg^RQ31H5qm$`+lBl&&W)xt${oug^W>I!P!!kEbVp-t+;F?0fb`KI1 zQ&4b&yb4qBneB7t4-gvbwRY;SOP`%>b=sv4^h*0&0)p7e#%K<)KH_3P2I4>ck$)a(TVl)JQ!!@;BSz7!pYn!)GNDb*&B$|7+k@ z99bXoRame6kt09;-xg5)3E;7H?|M<7GvIvE)sbE6;YG2#_`F@b>eR7bUPs(7$ zKK}F5Pix2uzTQp-EL{=>#-bJ}Y2K5A?s%^XTmnxU;fKr{uPR82vT-wsp=m*6}}w zb@s9Y{?udtUDLm1%(c7-fdnBq%b;P>h^S7D^CUc6)}0B7mfDIe0+QpjF>!vw@ESqN z$a4i$+}3+~m6&q{*SPsjvYF>rk8D<@@p6j#PDcdJUzfTrOa8dPEsB+$Q^$cZ>$1Wn z%3d?G^Y&xzJp0`Ct-RR}PW4xXoYvBleWd^5?!{Xv_h72zx5>%-rgk;ne^OJjA&==k zT1xFs_tmU#zUIx$FePRbBkrlFpQlduhlyUaodL8v;q^1!%YUTYgJeHl>S5V!I`XKs zhc*>0T^az$L0Cjsd|X_5Oah3UUBsOpvS|AS|x^%|Ta=e*zltsK%1q`2gvtZ^%sU`E=rr*m%KmSf?qF3io( zuVo#{R>&rvtBeN1Oz)B2`?9CBXdH?PzPlshii$TErz`?YD>NVt8ztST&9VuB8|jq} z$xFT5+}vsaOyFts!ETFM?0w3*6*8Osl8@KFZZfcGE&HWB##~#h)*M(Aig;v8PN654-3(uk7mI%K@=s_34}Oqey!b^NL;GEf#8~0Z9r667`ivow zwIXD#CwrOWlXk0{$_9aHdMm=KG%D1%p=eU6m z$c4@HD$bLMuUh(Uxula46GtE7&M0A!=`i)w>g>$)<1A5_pl>kHaY9tN(Mg{iUBJ-C z2nP1q^s1z;?VZq`{BwMX*(c_bbGBCNvGm@{1BS~sGBPr|dtMCf=hQJZfH?B3XB9CCas2T6yZeBp&)TIM%hq+ENtMDv zp(`&KTbaA6k%-oyW2d?2%BLwKsDQjG)RDo-&K}p*78(2GvpjX+JZyVB4HVvm(xg1E zlm?V*_BCdweTgN?-Gt#&<=YzQ8L_Yq@~FcLT6iA`)7_ZDpv&^xMp z6d-Ra7SU{#)|lsRo+geERv7k`wc<~>KPEQFs!IaL}}VkSP8g-$x(R<{5apld!q zgNdS~c64c)k_W5l*W<2ECiE^3zhHl*aj1FaGy5*>>>$_M_dwtkH&~O+EcE z#%dLjU@qm`QANSpQPxh+W!YxRXp=5APLy@H-^ta7c|S>?>_Q*%1OK-h6D)oaTuWy< zE}T$Hf4Wd!axKJnpt361SLMm-`x3~mE*{sjh*4)s#FO<0%Bl5oo3K`5S$%8bras-C z@Enx13A2pTD7#{fm@#kNT6UklX{M&cvKt2{XN}K&si~$*cBSQ2e_Xd^OTucbfQlG;c&q-g4Lua&l zwb>E=sqS)r+*=e53nHjs90UhSkCo+)WQVo?0n%yA{6}SB3Fh72qJt{u=y7%rG z0yPQGm5XLErEVj&mSe%X=AblgBx$zO&F(?M=-B#aR=5bJGLAB+$d*GIVP5Ao&cf;? zXmKoEGZqTY{wUkg%EE2edA>TmOkbkAT3S(WzcoF{T*2k4em{LWp8HKc{%cSZVs7Av zO-F>&{Btj>;czb11z8YXKG)7*CilHBKqJscXTR6=bRHiYx)axc)@pa*Q+TSt!k}(~ z{We+CJU@0r)|S)mk$TrSnasOYZSCHeXs~S(w-gdbT^H^swhzlr_PqWi>6ZpO-DQUo z`@`t$OaSG4^h(}e-neX50%4zyr?S^w_@rVXM}2pn8&~(F3bhDcMZ47pzoUtvGmo!k z{J2=!t&}*S+En&-`AspNo^{CO%a=RUZ{k$V1ytLs9!?l*h!>UunFr;f5sAVE0jEhe zk&g9_oJe@DVomf>J&yER!!GDnX{K)OpIA})i_x=7gdE*D=S*+>-Dxh>vq;?}> zY+03ytwznGs`nH^hep7yVDtNkE|Shaqu8wZR!PG;XK$0cOhGRuoeeNv5?_CM76!Sa zUvq(4Ku=?3NRzj5iO*xCiNT;`^ZU10P3h{)8#i*6%ngn!2XBR-Dp7(wJP32NnC(62 z216>Mf=qBkp8>vJ+C*-a_fG%K4XfVlr-L@@x|(CPMPz@SU@XTcSd5M7ACq}e_RGc7 zLOqx1C~O2g2-LH(Wq}aXLknw<#i!it%QpwOn=$l{%7Y88fhNhY_JLxKw-(y9yK|}F z>}QDI&E%~d(9zYMty?VNsR<*BoAjF|p?3XnTEOq~*;IAFAQ~D28UZ{|k!YfPNYj3a z)S&ggfS};)C$hPYT;pux_vSoVOPhNX0x+nq^|_- z!WygvD%;Fl!bV;$=y0&J3w04m-&^eVIr6XF8Vu}8_rxjQmP;MR6;ApFCUoO=D131> zQvrg$iU?Niy1odagj8t{9ZC&FZkmMg{ozx48MW| zZ_m<>$8S(%CO*~@mqp(vCbuzvj6gi zWwuh;Zl=`Qx?1{D-a3J{29cirDJ}6v;Ah22bV2zVnd^>v7fT!GwAo;ft@qPr7bR(u z$Rs@U(vo3Yg4=P8WU)jgoafBr>AP?hu`vAs>CSVq&_HkvtV|?G7``>ED^Zk_{v%k| zIs@X^1-#_ze+N3FSCmG!^1pM5vl=ilWp|f5WF+%X(pv!8A0P|BqdvMimD-v1Mq{mz zb7Nr?R$-JHfC&j*vL!12#^zv_?{>+&Dx`3iSZLh?08hAj(a1|yaBdf7b1QQwY5g7o7ZYa!OJJUSlEHqH&h~?>28DtEHv+0$YRp>kBG8vF` zH@a}1&|&*6PuVh_JdcL+7Js|H{n7i2eD%IQKKUCP2qxV`8L^!Nq2nf;6s4WZ@s18z3wL>=*bI(3I)Fp zrSLd7ifVnnXzyZVF4|#KbI)D`+R770A-QV>WOw*WI0!d4uYFSX4y^Rvz1qs$!R5 z_xn78mJs;|N;89*?vG5Oa7ph=b@%kNMMpNx9&1r#s%MA7b=ZZJ9F~Jq3}ej(a)LVA zv}sRI8wFc1ZP`l_DbK8uNn14d@ph)`#+EwA^CZ-`wkQu+8EI?rWlYT7_3ljq3N|*C zobnkSmz;?g%2SyJ($kC6Eg}f3ZnC~jlMZS45xVTXX8DA~#N`ULezLw4KY<~Kb^;4n zWaLwXb@{huGZEJ849#Yu5$uO<@Nc{t#ddjfM~dLBY2iotk}o}l0mXw^rp6p5?xmvI zkVta6anC^l%=+f!ABD7^i&2mgji|CqUArfRBh9Wp{!YH-7*SJmb3P+td4#&IuFgS{ ze=N~tf7@dF;PM8^?b{6d5C{b`r+T@=A``G3Kcc4Oj5#*~`rq=p`$}!xU`ZUPWfoD* zYVnZ{ESE#b>jneAWj3yRsmmY+6#*!6YXXEU=A})grgz`L4@M0|xwo87`=-!^iyx^~ zw%Ux(+WI?LHF;J3A-3WHd5F01NX=`H*XiSg3w(y7k`aY-E)EXGd`DXYdNLfnwE0gC zd3H@|o#(AsQrvf^ja=Gx6CxzD+m4hv1olHUa#nF}4MK8Mr>iRSiY!2iwmHEvuU()y z-hq#gPjL&HoZAzh*8M4R?&=P2Rl=7%X9gXL5}|N?()uK%`OiLAqx`rnAT!A&AQrN` zbsYHl^Fx=lY);*f!O~LCmm-lkMURiDAzgJasRsWT3#y(hgyAs9;y`ZLvvP+JKLLw- zE;5+SAg3|}Pz9VXg~MQGtjRgC)-g?i%`jl4tohU(bZqKq&{~;Wi~#y-g!QuDwJVd% z1>+v-5E+O5H8;6HJoyDPM5Si=R7-RDEYImtSGhY$Ks->n4lh&h#)kc~S*PKmQNrKR zg+DBK;yjzfH??mC8+c=K|J*-^!ex-jA{G8BDLmh%mZzz?m1}PAL-hH8x1oTfR^S@r zxX;L!N+tW5g6v*Rt3bhKGK;`DOiXEacio5qI^J|bMWVnADU+U*>`Nq&a2Y4(;7n(y zXN%o0oSb59yYuW~Rk0yuSd%DF(+jU&zn1oIx*#Jhoo?9DoQYT(Yd9)P@TgY!y=y=t zFU(EQf-)tvs7K}Tv2nZS4A>XF|HVZ3VFHbg4mQ}z@KFHK&~3dwl+gaZX*2?N9(i1y zuVBL4N3>QT3RPI}=B7vE)!MPr?w z(O?Tc2M%fCkCP_~`^JT6L5f}7l};C%+Scdy*I!6X}1-xA9=PgRiEscspyFEV)rV_Q-| z;bWnML}0cShqSKl%AHM7A;`Zj6@LFN0#@+WE_uSt4$OYH&THjA*)721}S5RuX4k z=ggf6nOylH3AT3rQ)+QPD52jzKh-POS-Fmyi;Yh}@D2`#gV2Igwmi?-qy-Nn=8M|%InjeGmZUySpUA`3r8?-oEm~o zx*aM;-zByNZh3e*!5B*!FuO#3l;@lOd; zq6opq(x{c4`eLM&qk> zzu8uulU`T7(K)HSj$?rf%>Q#XGny5&d`VP8iUht@3P;Yn*HvT1-7=py#5hV$04ICO zWX!FjsvAh-2(O6qO32BDpCfff>S$>nntTKW7TdSVa&UX+ws}{w7N1mc#`+D-!GhHd zXGkRFS4?wiwYF^pTwS1n9MQpdnK>i~;<|e!?I@-KxtRy#* zG5sWeknI5~%Lk%y)@Sv?FKZC)Dr&+{G}w&L%uHT12I7IdhY?JWYU4IQSR0XmQ3AmRxg(znW@~2`VtRxM^ytG<;F3Vzt z#u?2h2nxP1RuGp~Q}e>!c`dzh!+U+Uefhe*-C%BVacAWr}K{sAY(uQdFfw;(T*)Twin>i5J|xDL(0h%Y47NdaPw zVT^65aO=gPjYj|y#xb}&(S|`*Y(B(bVq&uGY;FCqiOqx-Sf3SktV0iZi>+Wl=2-&d zqu;jqauq1MPn4md4U^rcZ8AK}0$s_n?lpRHw-xxlsG3YU?WmoJH9?~w>7%26(CZ#vwQ+?Zk12V(X}k@{Js}{7#Xke>ecHvLwc^HM}+s$Fs7^X%TNyhz|$8`$GtPi;)RlZdPLb2)29i` zMU?Mg?*$*~xnwQN^UPz=a-hBnm~DfFQ$0?mWM-BZi?j7NAN)8D%6d^6MGJzs)?*d^$o&ta2f0z&I_<6q zH-Xe9Ix*EZ6 zk%5eokvLl?aK{?5c=I&5tab#$b==Edd3g=b=J8sTU}Mz{&+g3JvT7VVE|cboaAqBM z8o=OuY2h&pXqaOP+FBEp-&B+=egpk{!D zlhEL$n`W%XlMH+HV-Sww)R1A+KS+iRnGwBmHeIJ6)qNa6=j#TY6+d1tf z6IeW+r^I{iY%+Y2Y7$SYJmc`eQr1;SU*ACFF&#dP4!7Js3#HG_TzY zuF+Rg%Fog+d#LNNWdO3?#y=fiCw=6#I+r*?Y^Q^s(k7b{9`1^_d1;LFF>l&k>a*4r zhBm;$wDJGcji_dP%o??gH)Y-$>^G2w@8*arsCwf$guom`2*&n}Tf!NUAbEb9#ZdW| zFCw?dp3hbxrb#)&^gnNs-mB?zU3kYVWoVe9lVLTGBQRZ}=MfelKqpyYYKlh7=e&7O zI>RC?H=L1G`c`sd1F45T8b+JNegJtn>W3ttOiSyT{Q3jS7eCwpV)#nFjgH z)Z&0_{xtX8a>raAsp;2u?ZT{{HyF~KXOu*HZt~c@ZD1H9gwXD7I^<&~A6?dw|8HbY^-ZMa-qqs&bQ<4Ev84&sNXj`S++q6k%65C~FD6vz%j$TY`{rGDD>~FfzCg{3KYp|x zVPm7%jAw$#h(+EQV{N8gxPgy9`%509v{XL` z;R4U+%PlW=JFXU!3d>+=dSN#k7_Ek#FR)U5Ib&dBHvH3_u9}lI z@+Iiz_M4lv1ImLUI3IFs|B?_)Ses7 z$f#E>kSvYI7$QWv53}qD1;;uvvJzS@6`PF6fqZMqC{C;JD66tQEcO);Q3p2LEs=s6 zWN@xPWwZ~n*=4a5tAsAwDS=Rf;7)P5Rg?(WHeMm*-F{fM)V;=W(Rfcfw=qj&wJCb2 zAZB4(2cfUIHs(k&J1PJe8WZel$lzzvkbqPRJu{qP!nI19>E1KrkGT~4X$oR~JPl0C zh>pVulNGlx`z!>X{+xo-cfWyTzaimys5hQL_jk&DwlG!rx-F&9m*&MVj9RtBk4i@l z)9l+MY>&8X>!X4rg6<%Jww7auBJB3*s2ZaeHZC0w>C*E$_HFj{g$hXUCdy*^ez;bp zP3>M0)pz<Yd?^dQaV>PZ>t~A6A zc9aivg6>JugCA`dY}D~3&eW9ME&ING90F2A8xL`13R8<#ZN4)>GH!3WD+pKu!g3fj z-&*N|`}2}cTU(qI^z<#yXiF0k(1n5{SSP~ij~tR)cOU0$E_KOz1>C$NMD<yr}!~We}xB(oLQPnt&ES z7HW7;R8+Jn;q^J@x>o>~R*9v35${}0n{3?IEDhVKWkO0Jfiai)Is(1Gk=4?k#E6YnKDlzY&gsqQZ|)D~ z*WSeRP?g$G4sw_6v>XFvhB#?sW4xxOCI`7&A(&L4ez|9RN5obRy^9>|fWBajvJ zYO*wc$&f;%O-X}3ANMm*0WMS=f#(DUWl*OQe@tp~eib};%0#0bnhYPoz^luG? z1nEv&4$ICfn&=qCrAg68UI65v1XNsVu5aU1bUd0qt=-NYM!FV$ExbT&W| z6%~5z)<|#c@z!jd2<#O@uCXDKj2v1b5~pnR-OC;$quF!n0owUq2|+|=4@&9L?!yxo z-IVTkoNO?8>#(&0{vQ|^Kx}U1YS7BIDY+UL2$`gW8VZ{9tEfs#NlBHd;<7(a`h*+c z=GIJ{zCghC2q?c6OWm4dagd{3eYD@b8{=M2SiGPeVxppEW0gA)gQ7D3EY$*V3n*$C z_|);>WBUkp$R1>z{jExfppxv&q_2k6#28Dp=KHFo16dR)PiV!_=$(-gt32tOYHW^x zJe-yG<@h_Y1(;;@+ahJ8zmF-Tt8d)Rm+*NwY{@&ZK{Er-b^S9N zkt_jeTMYf)x?0)MFx(}RevekVW?63s^F2ecfTJa+5S%Z#!b(r0sO5or@M}*Iet4|A!fhb=9&-Z2l)~*g4XP4o^{=T?$Lc@=G9ViI5a_YWi;Jp z|BVdO9d_@dhA!urr!|UsGA4oX7w6dtcMyi=s<(NZ%sjKTO29IPc-V2B+Ii=zkO z2%AaGKdytBP`#d1CjV>R{m_86`d-4m zco7Lx@SD~6kt^UqJ3FnLS|xHNdoGOVN6MsOm}oqS5w3dweo!gl+0$uIrkcAq$2dT6 z414K@x3^yiQ^RH;shYc%<9C)n629;CtJ1dPt{!GuYtw4HTyTt!3H7211jrBz3qqo0ikxV;H{wr3 zSdb8~MZ=(Y9p_)vz^4FRGMKLZ`(lpIb z-*;He;y}Cx>ZidkAL@t&z7(v*2NK#h<4sTs0p#_D8 zV9PV4iYK5xh(qRpJ-};!={@&J4_QE6@qD`pz{4-+3aS~Vh=_>wHB*3qjA^nSk0>Ba zu#w69StiCiz$ffGBO>+-GZjp{*4Q}MleY3E>f~})Rpf%=cUrmidouh%I{*BZCJH7< z(Oc<((@XTl3+9V}naV(*{?3|A{=w)R$ z(KNCEIQ>*!11aaH6g7udf>b$M%-Ex~4v*#`DnV7Xvh{_`=}0rGQ2X@HwkwC~kGkFw z9J?>uLl9L6=REQ9KdbQvCUTwp2rU!;J-vQzW%1Xbm6ZHPF;Xz?HJ`Q|=d99dGc)Dn&zk;9G>^#7nzX( z=lQ{aKm)Yp0Su>mIRp%EAd*lh)E_8R70bx{|v3Gl0r1u2dIT5*AM>gMa z){pjA%hYX%1=!Y5%Z`usTlvCE({snQ0P2<54uav7Fkg7@ct|Y67(8}yysE+%zR5D* zn;znx^-;hLi%9$SgB=;I4vu?zQWZS2F^KO0tK<{8kG{H2oywx-z=>Pxpsk7ZAlra;Ep3i8UBqMO>nCU(PGOXy4wUvS_< zdl-y7IQl}tZ8mp>5HQXTw$|yB#7lBWY-0|3dsvKrJRT~b7XDoOnF1a!=FMc! zBcWWjtH+6SF8Jzuan+-;lA8DSY}$CrW@C>VeM=_7)pgxu**fjNVm{OF3oz4v8?oWB zQYOB)IZs~E+bB-kWuPww(%&dD#jx!6#K&>4+8s!X5-)B#8Po7_SNfZ$w}^vCNjQeW3ph2&Q=h%Ehr?Z~sld zq6y%H4+^Mnb`YGHtP0AghrD&eee{$y41A2=jE-V83t}G}OKP<3#b5uu#r@-m2$ElU zRBHVJ8)8=!!By@%E-!GX|55>SyWL@VPx%NkpUmRW>Xd(wnY(l(cv0X`1xZlDQ8=8= zUt@yavMSY;BeZ@@N{u6><2W`k9jYuBgbY3)^_PYUO+pwn3nj+g7a7%gXD5AyJg0)* zkQRT+TvfaM=biFT;Jwa_XWf+m&Bxon%^V=G_cmr9+s(fK6_0vx z{^0(zsZZW`$*bPfgS$M^7^cPjs)mve+_(v4!?vbd^C}V2Ox2CL<#XZ`NzIpLk5`27`y~@Q5{D>OzQ%BR+ zix?aST9bcCH)2%Y`DovAW2oA#V(v)5Q&z1byJXbdQ*79gx?poOF@W!mQ(4(HlB>F3 zttZ^}mxTu42j4E*umr`ro90(l14gQ--ijzJVT1E{7&6pNxxI-?uRe?tR3H+A<=@WY z!eAoS12Fy(9-dS*yh4eIf`&>0shr*DYThwj+r9l^{U;_GMW$h!{3~28u3tq7q$`_9 zx|LK^@+Blh-G+QD_1MhPK{dgESO;3+Y&f$~9CnJVKw5f;Tj-Pg?{;OM014T0EY5YE z7W@3Az$STqT7iSfh!1R#DVnJ1uqXNTZ1gP}>k7QgV|tn7{(+#|BU~7fNv6#^()DHG zNAxXy5%JDuhOqK}RazQ`(7;Xw@3+3|&c%;)#?b%dT%Ug%h+XIS$BaXoqAy-Z3$pq# z5ekeO5|lAlr3~>#a@(4BtmM1yLMrT{Iht7`*jHSb%XsMQf56~JeM{WepTXu`j$RWi z_f1U&^QDYtF62A!8kTVp%Of0#TxTM-^8qoR*{XWi3kixo;5G?%-5xSpk!^X`(F2d@-sl3i0}2EGo_kExLTp|T>Yf$CAJa(IXf&{2xW$z!{i zPB-n#iZzr~@T}Y}IBdd$`I)RksB-j?_5GU#7yE@!cRd5NN8-dz zp@5<5y;7DeeQ)7#O8wh#3Gf&S9AxKLmPoY9jZ{I}#ldoO=F*n39ADovy*y7M)wNe% zi0$_^>IoXf!GCg~I;}y?5Yy5Q|A-S)=;wKU4jbd#o7;9dbksg=Gd^{7qU(ERxy@}Q zqja_c2Q?f$S%i#M)obMsKrVT{Wv#r!F{MklYXtXq3+^5apBqtpAH?EVn`8T{N2QTm zrj3-S>9q!uFste#O(Y5S5;2ENx#Q{`f`AGU-NRM-l|I=j@hLsdiKnpeo+aAUQKyDw z$m${PTdYF1jrm-d4qcVZz2wd3n()Tacgic3hYn?{P;Zvur8{BI@&oNUGS{UH+(4lX zX{MXJYHGNeW~`B;`!@9lPo{Ap$LP2cYwo<{i@I&!2i+bZo@FNuNg>7^;oA0necz1G zjB!UjaxE6ek5ezsjL4+fwi8iuSR$0@9;wo*p7SC0$aF6+7W3vFfE-tAms4KL#C z(SLy{RkBh=NnSuvke%U%A4N&jgHW;3E6mu|>2WSEOmHSx()*U5a@cv|>ljP?LWAL|J_|lc66L!Y3yuSqdEOvVcUBgqm8v zkdCPWJtgII{6<(zC*?|aPbNVtO{V;qxTLg$gM$WanLEbggN9Q3t>mnI`>!#AmAZ zWe?i4dzS9GmBI4sP5QISu5fzmL`x;mqAF}7J7-HT*Wt2&kaS{husrMub1j25e~+l{`BcAUKao4;vDnw#nyGxBE`bhZd4j#@q*-N z-}D84EuJgi3J3B4J$8O862wzuI6)COdC^bQRXNiNo}hGCs4L^5``CVNv+L-7roS1e zv84}4=;tuw3k48Hv$rccTHIO}NGbpe$HLuxYdf*IZA^Xa&SFxvVWCOFCreI6_yymG45+B;88Z}#b4xy+@1}&bH(sr_YbkQhEf;wP!pF!EW_(S&Ch@!W z-+WZ)9+Gg${{VC54+i-AH~u^EF80&Y1QH>KEj)W)WNkx!b=G77a7JCXyKKMz?!BMe zBl3qe7lzNf3zUH}X|+Nf?|*%)&;YJqxEcNRujt6xKmRj&dG_;v-f`g(@C%4NS#O^; z8BQATv+1UzW=`qXB=Q$foYf}U3;WK4%1&oqmic@lI1^izia4f`|yMPOMbkkgnTWbeZ(& zHIw!REh2(2*g#ibxUcq2ZRqLVo(u$SCuX_xM@0MQps^DSmOhEW5h+^iY{E)39miul z^%RgczlZD|cRpOhQ#BR=^NZ$VjKQw!k96CXQ$x?@4jScmOlW(Cm;@XoJHRU?m4Z$8 z_@)Wd-uZM#N#W6R8lB!Wi;(TB?55)6r{BT25)JBy=r29XT95yF-h86CPL2a4OHfcz zzS}6vaO}(y1qB4^q@Fy1f*e2I&eqkM4uguF+@faL+MAaDzhF#zfTY+al(?CJWmH@N< zCSS+#FsBWI#&xUekZ?x6Rqj8Tw@iG#4YNX?SS_LidP_DrcU(*Vw`AS|zBG=A+wn4_ z3*35exRM5D2bJpQ|O&i*6KU zRUeQyte>0ajmu8~ycc8upT+cxHNYc+fx>xL;(k^Z10lzzjMK^xZO+Ekt5+chYYozX zU9=c&8$Zite4UVRK01HZS$Cw+a%}j!>pX6Q-Lj@(%$FpJc}VMhl0L0M<2#`8O#-2~ zn;e%Ao0+s8kKDu0sMN&Do+cpB(Vebbx9>h*8L-^f!Y(Hv@pf}nK#<3Ag~4LD@cQ9W zC;%y>wY8%+y0&!twRH%%|G-Lu@%|VY2>G-HW2H-HmdklEG9HXfE#@T$==Ufw^(5rvOv5Yj=O6AHjicuJtHCCpqmPN1Wpmx z-}7vYJfjjal3mi63qQCU5fPDZF~UL1#I$`ri+g#7aYrRZK8gxgU=yTrZgT@JR?jkJ zw#P&jqR}k3+El>o#^At&J2g>Kcf#x{vI-!6NGX!n4O+(G7ixxd{XP|G&_c%eDFZ~Gx=v7i-*Dw zDLI(>-QD-no7Q`$nm}o|*TKQTcOxT`DBT>(qa79myHKKBoc1DU0q00)TSZ9dgEohI zGzA3}@MxHeW@lV?K0Mf*6xsa-G>fY=3UpA@L4lvf!TYl#JXRg9;?A3MPXdk)Y+-J* zf8{)aGx|gO?=$+Jb5Za*W;CeM$!}uAl4#eFHW?WF*gMwB@}+`7m|+{P{`cq4Txbw{ z@emGYECs1^@pk-SnAz39J(Of26w`Ap6A5R~vP4sYH}g>$tLP$`ZssTpL$u0CfF!5i zhnu;?uh7n7PxMN&(E7vefeM)MnqNiHw<=ZsMu})&wWNVD;5cElA zuy&3)wwafCInE%(3bL1;+W5*WCw%U`ahbAN+!ONW+kQmII)lc&xb&3+WotrRH;BJsYn=4O z=1s5DX&Su|0JG&uG$cE{FAK)cpsjqO!I7A6k-vN(L4bc##c) zR}&bGVcZ{BF75>p zcBVo`ywaFqE^nng$3B=m zq_$d#s6P0f5@lX-gpT#zV1G|5Zhy66K))%ug&bEvKliU>@EVBg8{wGyXSZ#l3tZA` zCL42eH`STbP13uwUTgLD9MRL#TDIg7e^YWDPBs+a9#UxHEG+tzx}+riql_wfi~f<1 zd;yG!#L*jjycJflQd!H<<4c2c4M6ODRksH6R-nm3)TvxT9EhqsyH#qPhs-o-6v{H_ z51FffRA%k0@J11`KdupUI&9;dj#eZ$XbTYF(HTlci8day#BsAT8196Aty}S zQiXs0Mwly`#n?NSiK7&&7??}fWMp_5sr%!J#DoM&@@D~jOpEWBie`KJRxxp=aEN(Y zYa28)KW1&8=B30JV_rTeC>tj6K05RMs2G)Wfk6>pFNjziedip?tI}`cHv+O)$Fd-) zRBgSBmE@ZW_mm@!2s?2&nazyqn39458Ryt0vr`iT@j$t~8TWir^)&(l>U6`^r1j=X zr?^3~7V-doDsj>=v7^;(Uh|m_PCbuh)Ai5ql#{heR@NtlaYx9yuK{A0$+|M6CA*|4 z#&OhAPm{aMAV1TO#5lY=5Xjn#cf>*f)ULn0hXa`oQ}^D^-WT%YUwwc)#F5@O8*4_n zL1W+t2n6m90f<*#%l6?%NU}HqKyC64Zxa$4Ao7vLD}a1^3Qii}unw8=oWo(TIC@s_ z>dotjrHlamU3EQgCgs75zN`7hT@M#ochggswfo+jvhixZUL(U>ux?^yI%B(9zeP*; zSPAuy3CL^dULJvw1MWm&%jfxVVGSz5u-NMTn)4hkbDiXhTCsx^%PE^&ow?i)mc$2g zk+>i`n`ub>f6{RQm;V1g9XD_luA>X9W&Nf0rF!F3Hq0%C3`i{tT|K)WC*eJDslC`S zL(-&8PctroHR4B~RwjA+9`q!+y)wp9F);Yf9vi`bz0b-dKX`kkEr0)JSWYUC&Sc!` z^u#j&@N`ZK;3qf)Pz_+0ptq%U^94myu0%^?V`C?1h$h26+vY5~sMr-&RWnmJ92T|- zUa;u?vcfrS^Wg$1$NyDkZn40-w(O_CJi!4FrCpAwA$JWhjAldK2Am`BC!=yn9pZHg z+W2GNOi2!l+fwh*_o%7KHN8`AyXvpK*u)#pz8hW~k|N}z7myqIKn zRs`s;WEFt9_^h6Ok$<^oF6hK{9$Mo63Kal9`J2msHsbW{Y8JukSISA9aV=lGusV-{ zfl+G7MnNGG$>Hbdmeb~$mS*hz_HCgsMUf1@ahjTVA^%+TzPDjplr)HA@6-C99?ke0 z03+x_qx+K|^3$F5Q?2RfkQeiPpq^9ll?s`I2U(+pd1J)~YZ2e6Z6{HrE^+%NNiCf_ z-Y}HcPHOw1;k}}wqJ#U<(e!aC@6#z`6H7G3GozlS16@B>)2;a4{iitf&*2qB$LCs4 zOH4L^GF`lK`!out)Oot7~NUq76dnL7%qy6i1}0(ibTc- zh4dvgeEoWLyK*1)>XnFz+mzz9n>Xj&%Ui$u`d)G}s*eJumVPdbWW;33=%oq|HE|E^ zFg&(LhV~4mal2e_+QU;|^Zwi^4Ho2>#s{4@wx+&!PXb63OaV4T2%%M#dr?yp!rf!l(EFJvR-Iq0>CK!5kPGitil|06TUoRshkuWZ^gdEZ>(or=h`&!PSg@nm|skw-59wZ2((K-;Ke>z1{TYms@nYn5xn z+>tC%QY8Pf)BJ!(Jhs@bq@7)%+dwl>(s1D#^+@|}!ZOgfSR#>A*LLzVPEl2r)_BG% zl(wqwO5GC9bYo0lVd*RR^nf0Rq_y>1291N)*zbYzd0@uE^WHN!ojia{Vo6;81m*wP za6ytZ=bn$*a4%LE&?^!F+bFq)hDPx;sP&mtS~|$ol}UFRwuumM)o z{q3V3C1q(e8TV;hV>LyWg3Z_)ZXNCI#SQQypoQbbBg&DbM=fCSi1|>qzLyz(V4lmZ z$HtmoG^Dq%YZS-QvQ_m2U8}t)qY}nhTnt{!b2-87&E;T=mwG@1#zCVdomnQxDbAIm z0l}R|C#>)%c?=9X-4y`kpxtIVjmd$v_54ECu1%QTYL{|O)e))EqesTf)m!yu*KQFw zDg;Ute7MniEW%X2qclI5SJ0*(xiu~`VtBY$1d$oA%wJ2zF$_o-Bz)d%lJ7pI8SrU+ z9b%Ev+lQCKN~p9878Z+c-uvT-YI^!Y+WYEyiTMsfhQz$!>l|f*ALZrRlb_{h z=RahIayrbTgmR>{KgvPnI(?);SpZ5I*7RBO?2!reqPYiH#5gv?t*7KBp<=Z0PAX^i z+0*y>bu#_W>H7cnbUkZ-0Gk9DASD0N!>2Zd|L+(e#wuuacisIKyh;f%h|*u|T7{js zZEO%E&V7Jt>vDRJoYX=CWXNx%S0=SM+s5v8)?jSHyDBRyu{g~BfXpg-z-;{w!Gm5c zXyvGOQ%74!XD8K`gTKrrRJ|ML0d-Fh?^QF&rVSz@VIPT^T)dH#P`h zk!b^^u?md$25>aA-LBjq0ZYqV)bscekFAA-gwWs9DxaLwmDDxzcgG8osL!nX_;GCG z8~70lTPu0@?wtC>^&totx_-NV63Y8Gd-;Xn7~nK!XjzG!IdD(i<{#tWAqb4gKQOdB ztJU(?G*8n4LvFv=c2+|5uLbf#9vK+(ZfKSLuhyIAPZ16z-wfALz z?SyB)6K@H&%f4*R#It+v>1heNh949pZKft+<@Elm!Ds&L%?7An`Ig~X72?0F@)~Mj zFz#(Ej@| zr}`oenKODOaq!x!$ZdkN_fa<={B(?N}NTJK=p3 zRiJo$H(8Q@zyDu1`^%nSkE?yr9{AS|dFD^SX9PRP+bbCNG8VKacVH+`L9#1kSJZ?Y z6cntVgovtONYQ5p$kq%PVHPSWOk`S&nP$JFFY=juS)|FSrr4bD)+fJ$o+e)D{Qh0( z8`sDxH)6iZ83sq_8?uDRfVh}fgv%+D)%>Jq{~2EzbTD}hUbIvG*ZtcE=|?9_JgNV!-}m!@ ziqzHzJ9%k~c*GA;pg!b*HV4n^2rrTU8U!eW5qwE|f}iJKD$bW;1e^B1Jds{`O9xt^ zXwN%$>GXSlc_~;wB$HMs=J)JLsV^X^NJ+-e43Xgvg(qXQ8~mQ1kHA;!t|^{{E6&4T z;Jv-z(NYK&Y#vh$oH_4^xMt^{cYI3%zR_jNJa>j4^5s2>vQgb2j#24(kw;|dhHdBd zpucrPP@t1w5n#@WTKrxWKaUqAF%{yypN0a zd(=c;gIShVieq>3zQ{p%Dl3@U(#_oe+NX%ONIMdftu zvixW+n}>4xq#Y&iT0CxfJ9=9LC#_6+70)j*5^isGeya*9#kaVtP>tC>aqkdbS{(GFDW;kd;D_d0eYO4`08h`G1=>i7ot?=d+qA_tc8l)1?%Ul-Uq=cQ_} zMUo>-cF}amGsOHTB+Yt7E|c=m7_}aods|OR=6?8;N)@72mgBhH@ugpKJBFl3FcdFz zJ>v@&+I`x`M@schA|HH(0E6yO$sY@+o8p7i%iH{}A0>znD^?Ch;)To(IX(Zhp#Y_4 zO>|@IgVpJ&am5x*j!X*VemyDRx7*hG_Dv>+3NqJ;>WD&j=aWnmOnbyobeAvp)j76} z^v5X81(G;o^ERMVjh4Gd1L@Q{D*8)4M!E&WQlIon<)USr8zIY-5sPga@IqhaY>7Wf>EeouNG_hl`~Vz2>ac(8teS0^;=!9LdQnnRG6-2Un3&%5dw@r| zGS#7Fem0B4=oXML;fm{X8-wrcJa<})W0s8LcXNyVzQmBBlPXu{#uq~R!iUwv%6_;N z$Pyr|*>*yeN@7(u250$=$GL{&_pDxSSYe>j$ln?NwI&4%FMyev?daux+Q1f+xyYq# zJ}+!^2l$cx5 zzT!R~$ISC)EVp_k4y(G{VVO$Lb?vq5k5Bt6<(L!reU_ifKUpFcJzhr}xyHq?3j=@2T}(KZJ6McLh>{mE+eF=~Q+DuIg`LD)nktDTbH=vDKL4vqE$@~10gw-4pt z;7XdBMlS7i;I7T>PaBd_>J!7`d}SIZueGDRP7mgPXX?sYS?;3x#{PTFs>Q(WHf z)$N-mnE!!IqAPnul)s>aF5=A=fu3s~2zlvpi+ZX=)+tvwSd>k5OIth^_vF>geIjgB zDQ6MnAHzdfAhyeUP-$j~CMcJp@a8hHd3F*Y6Llq~j;%(gM2N_F6nd0r=f?ij=(m_F zD+&%V07@7v`kj6JpmyDObJ%Lrq!9Y_MhCwwXQEqw^*$YsoJT|S9;0S)s62EeHXx=9$ zul2W>>0uq6H5qY+Unn=hPNsw0yYOQ7D=?b!g&~GC3cUAA&vlI)QKaNe-Q_OYR@pA^IvK6EB2?KyevOeE z*P+XbzJL?fhj-O9;=k<%=p2Au>3CMqtIk>&2N&i%jSknfaZ^69g)@Xid<%8nTI98= z(go>Z%=?IC`(;}`4+Ia%l9-|pC63^{rh@EnA%s&)o1<4hB%5^&a4inAvdq0%KT)Lm z-e%WaDMRSq;^KNnG9|$G%md*I9j@5qF1j(#*@sIp#t%Yh&Kp(CGDOTzFh?M z)T;agP_w)^gnMJA3|$P==M9Id>pnqq|)P=z-9)jYDrZ2Uc9N<{iJdR~G7!7pZ zx*nU2LF>D-aZUNttAm$PhdKn9k-}NK7jHzOwBhJlR6WgqSLhpe?^9t{-KeJ-nN~)fn>jqIVVV9!ZA*} zH#_q-uXC>)oYP^H%VZ#>sB*jhHn;mh>HNfFN8Y`ZwEbBPn<3s+A7?)Omo_|dM$Mx3 zeXV&gJ?ADh`Rq8lTl~AdG50xu!RB;)7#1swd3f|afWPo4glb}iU)}TVY;=zjk%K&I zgv3NMF4aXT^axJ5QjNR!c+{j6@w7>FGPjMO@kN0K577Dy)K>q4BDZHdX zE{>m)$Q9y)6}SC9ojbz0_&|EoQM2GaWaYhN&+isU)J=#Mo)5FWRH7h*oe6F)Nbu;F#d*T}G zybaIEHl*{bQ02gNP=lYJd8$uY>9he7jBki!44L~)RWChEZ5&m z9P5H$4fSd@BG$J8D9X7)+*5*cH)7nXS~p!tvvnCg%2qo#YSw=Mg|9cR)y{HVh{;}) z?nZqfzgxUw#TH|9EfX4dc*0wp5rwm+KP-Oqp||x~;7sij=ms{B#91EHI~Qe|DVV%4 z>dU~b+FlgP)-0MrR;g9STG%HzUc-&bR!Mt$pO-B#5miJF(J_&Zo>Tf6E1oTWXVW)z zDjBAiw`g3oy^y&by1<^%LVS$y$Tw^=)z}1~x1BEW>4Y4DJu{~GLn0K0N1OYpCOwAv z9>=jiNw`IN`ud@c?VH9D$Y@qI!Y)>$cew95$|q9swZZ1S-WBq!4g*UFGGv{w?* zydA0<)f_e>pX8ePIUng>W4dwp@y*eJtVN(9y!g0OHoqYMp=ovD-cqJ54pB-|b(M=N zu}@>Qb-8GgCn_5esUUI2Y2y%>)*?YK=y~tz$i?tXuY?&|?a-aWUFn$6`3&L+Q!+)f zY#9;q^F1YHEb*=w@%bgCDKoYUF4Yb^ynUm+>A3GNC6TYsx~$|u@v=c3$oXEAnD#}A zcQA&qSU#5_<^9cNBR!9uwR1y7^g81j)vcBtPT%djd_RDVq>MFgA+;YyDGeKB5AV;` zp`YjNNw%GBPk;vI!iScQBDog{N4$>m%!ib?gJKwOahiX0ixBu<)qQ1HlwB7t1_&si zQZERCL5D#}4-F`51!U_OZ;`dfqu`0wt)J^^+8aubq9wWapD`C zBnB-!yv1H_lY@b9DPJ;A_|``yFT2izdX*lLF2)YE?QKbB4N*%#R6_)urgquRhHN(7rKHq^gK~xb>HWfVcjI8%O z$6Cm&aQ6yZ^0bD2I(FlHp~nir!C?ILHd2rEHV9X)_@V>5?&OUvoe1RCtsCg?oj zZ|(LB&zjF9YM$9@lxFkVb7ifka(M@iRlHj1JG;d7MRvX$v#dwql(1W^^$0&Def~4(ZSBM85$4MhET=hOcJ-Nm$xc#N)>>xN1Ly+ ztqLd6UwPz~c8tO&Q@Q76r3dnw@GyRgYoc#BO&o0e0w#vgDGq_3MtpEcjqzI-HL67q zgaJkPA(hsd!Ru=&(6D=y%(i(0godNYEg>E2I!kH_MhRVGzX=mz0|kt3vba?g+TMG% zQ=Yt+%KSb_9oX`GKncCo{5D_GkYl@vRpXK`g`+i~r$T@>*v6$tXlH%9AiMPU0;D-+ z7#ormJ8UAK$1yRwyye=W=(Mg9<6&?1seX0s62obxGDNx3heYLpJPn!+Zbk^!x~wA> z?H-j(oHH@xBr;?-P1vdaLJ9vi9_K;9GrX#kYJX|hjfy7Q2m}3!&S^5JVy*PBS5i&8 zm?a;o1molcKYvQeRJj3RXA~+W-E8 z7gu1BKmggxp{B%tOpfsBg2Br%t!7j^F+qcFDD!24oU|n3#_?v2;I(~bgovK(xWDA% zfr|y8%EWW{ht~0key%oGS3?GF4^HKeTy0(&Wd~-Ky~Wr}Sg}%~cGvOMStN(Hy?3@; z8(6kE-M*rc12T+boki!iP;-4;m*+C?&kZULegMR1Y?6}0Je+Epfp=OTca}MSI_qUN zFEJDf3RIhnJhz)nJYNm+J;rBqh+_1ubHlrdnzAZKE4l|nDa0C}26?z{)=1@W5x1|r z2ybP-0K{MeC9`;#@=)GHql)~shAcT``s&;ej|%$kAj$0N+%QMwEx3KOXuvT>!Twq! z%iOceTJFu2;CwbH%YjbTE7UHIwX#H)4UIYxj7cb&tm{^BYm)DuMrj0xCJT+@w(+Qv zdBXbBXeHGJ+EWi0HBl z>h((|XK&*eDZXQT$5R9S!?dJ7`3*|En{qa$$J>8FjWK2je}{J_G&5|=ZDSMr-YI|b zc)Qu)n+w-|FEvLg`!!BF7WLj7Vng&SpEo~fuCGw1z2u0dbDg8e#qONxW$1Qp^Aa>_ zQVM;n3!3h}1#?cYWxpGv`SjkXlsLwvPhgR@XR*j_O>(WdQSE8APV{Nlu|PZK3tR5b zjVB+IwbJm48JVc7#~R%%94pmpm$1X!Mdf2diK&QRfC!tplNS|nYzON_YoDJ_z|@=5 z)hv7tzw9s*e3^jXv^gFwmX~ivj4-OLNQ-!PrNFT^;F)%@Ej0}lQ$LpW7>|irR^{I6 zolwhIBBqCrA}IT*g5tGyR@Smn?4&F2txg@Bxhl9^B~vdrQ0b`x*^1Q6F&(}-*FZeX zA9KsZ&oTFA--_s$*87jH+5zV9j*nM1Q*Ne52KFHVEF$p0sHhI56$(3Dzy|{Ip`#9Uo{!`Kf1QM*)5Oq2xUJQEfbcj0T zsR#Ool%`p)!g)r0RK#A`ovJt@14JjIpPhJuDV1&1vsV*(bkQTXjO?lflk{~W?ucB$ z(A~%Laevr#%jQW(dZSs#yOsEbck@PHTVB!e$_DsH`XsseKGpjY6ZF?YLQdHOMyCqT zCaLDB6*aO)e9v;itmy&oa-}04RCOXqa)g^VuF7x@BTSTCMhHiBMOW^uX!)f=jClV! z9ugK>-TKeZD05%@zh^{Jb<@t2qhS<}tsxd)yH!vIeXQY@*KJsDw&#r>y&`-{q$EkV ziRP&9+xNa4BmYKi_6{AtUm(P^&Q=2W$KHq_O89L#1%SAPnA*_&=M8^{w+}1;>{#l% z#_%&4zklxc*6xYp4|1TB^p?(@q(hQS6boSMpo4|2E_tb+R5 zId4=QB$=>5s|p4~{HoyCuSC)BG5olK#gfLKkF)u5Z-uSO{b3sg+gQ;#n(bIw3_8p zFGaCXkZNX?%a2m)zfH&P5x;Is*~YWeM0%FSclaGiMzd@4+pgsY7trA@eE-H}|Co>m zjt3|br{Y~R=}fT9p@Vdj2~NON6NK$fpa1@@x7;21rDRt_pI5|olNsR}G0F06F2-hn zfn9UYuaxz#rD!IV$Jw-|e@IbsSCQg?8&605lUd zAyPAiCJqhaWXxYI_tz{UHdssu^Uu3is!4kFeb=yM(O*rlH!hlU&t2-h4eABzZ+Nbg z^V;P0T-S37)>cO9x7*YYG}zgL!iq37p)LQvZ_@>y*0xAPF+a zKZGlTcq{DFT>WFQkfuPiwQ~XN5>i~F%GmYWwcb1 z=n)YbzntHLyjJ)6lkAaf+g@9XLA9FN)pH#uDf}p3AYzK>iww>aPAUu~94ff>A7djY z7ld~yo-~*|lCIvSedX0%1k+`c-(_vEDVi(p3xV0f3ZRN}CN?ciYbLD+Yuo96SRb4K zVXFeMD_32@%>xeFq$uS>;`9G#JshL!h{)>`K;0$I_;>`JLK8=ubKse3GY+>NY$#<4 z;mk|K118&*SaBR`$V&R&F+99AbtR(R42864ahvGHj6MP~t@z@nN9Iw4)1BDE>KiM# z3zfs$b;ep6>gw+>Iup&G2obg!W(?x<-VD1D?#|h&Oe_bWQDBwiWwQE=w9#apO3*D%Z>(UR(D-nm6KQ7K1vh zho`>{VCEJJV`qpxlfvhC5U&)kn|P+p*@2kV;!R5Uc-Q>!oX6T}$yuL(qk)w`y01;T z4hck>UNx3RF-qz$i^1Z#G#nEONhsZQddIqxp;$lvNXEID%4%04r*vQ2drFV5l&Yd3 z4O!ZlNCu6mO@-NiHo|YaF7d@pB#sVq1GQQglPr@R9`mm8p_^NuQmPNV`tDjYju8V) z!@R|Fw7)zvHqP#5xoz*eQ5-khTy8}nh>YEDo%?HKT)(E42SF%E?)OtLR~}Q0|Kj5y z#2BQk((v7a67%RRnp*K$ndd7gpcqGsj4qc^c<|bO1W`GD7O66^Jl&n1MA1U9M7PIWk7R7o z2zcU%n?yK0W(8A}_=I9BlsV2)UKF(FoalS^668M`0hd+1s~_WcJF+Gd3+5y7DHqT< z+7-4ON529C5xvhmq0*jgIQMkC5w`}QH=*vK@vWUzENrhWDJC!qDlK_(&JyaU=~ml_ z`&ni3i8Ay5sC;70()fnwVGL9Z@J#_=22p(C47;h8=5gDyrat&xAn@s^tzGdBuF@oC zF`nP)+Fr9ZNCYEEO;5ihyDhxs0vC6JfwZHn?fKs4T5F(9vbXhZA5?v&e~x2kJ7ja~ zAxE~ukjG`K5eej!hgNk!CD~PqSBvDH7h9^LmQb6+wCt*Gnzk6_ifXBWmudj@hH&eY zm>)8JDj)R%R%X#M8e^lu62FIfD>++k2Zg$Rp%A%(?#JjD`=D#1zT@5!&I25PA5}pN zYLC-uADHqRu(eU+f&$NtlOfkTLU6mhs(LZ71vv=_o~dn1cY z;G71D;Rr&tBOk~Gt!jJ#Me^k<%TyGBep|gd9^wK5-S$sBoGNX56~A8a?RHq1&U@0_ z_-V%?SeOTBpvJZ*n_E43B%@=(=F|$eD!YFY-7nUXRXLIvjxwIARlaC;;|I#^Kci0& z`GIrKSG@gZiNJ5)1#Gvo>ndrk#8qvN?ff=7@2}Qcv&YkIWq*sLc1#RrKVER4_Et)5 z43%~XsXV>)kJv4rzGEdZNbd<3BZ=l|0uzN}&hY)s(fON%X2U;9EHXZlO$quacV@(C z_!4I<>#E5Va%(*Uvb|q#rA)+zqnh4^E;+oGZ|^>KjsjA=eYf1*8lw%mQ!j`N`y!_V zJ@wrgleYZ`%@KL&ySmeOAnu!rDGg?lxT|_E+L_oo-4BQaKo{eeXZ;HznBCHfMWBPS ztG)pN=1kK3Y3leMTG$2AfrK*?B}|`F>ZBGj2!n&sm&_Wx z%6s0ME6*v8ve{vZO8rdA#z!r~UTb=$QAw{u+*1p60(b!(T0)=0;LB)HC&5!U4!OdV zo$_E3=<_M!0kDtW)jR@IWm0|-_1@U$rf*UfTaoQ8ud3X}N5kpGlekvPUzS?4(DOca z%2Q1h@hs_irp0r|PR&rcM(41JzTdF$m>5;elK1hJE*)%m+tRJXp9tz5;^d?HtT?(k;(MKI@-3{MQF^O zhLcaE>9(XgT)d-tby6~C3GbiV&DktX)_z)B)-R8p+k;*$i%VC#5IaQp<+wnD?M2?9<=ua$&=0sAvXW;XMwU~^2D%CnHY zWNr^*-0XsemQ-0jZhEWFUzvtxTr7Hu6A&=cpk1Br)&bJ;<4|3J)a4fBU6M@q+1+#& zp&T&eiIadW?rqX1&kx8lXL^e{_}U$8g5-G8WLL`*oBF%B2--!YzLHPV#619qbQmvw zt!@r08!cgrEdF4$GkUqt+;&*dB`9rUp*~tFHj&R@5Ig>Lys@fSKYRqd$bt( zZ?fOYkO6xW3H5{SODs~pS7v4M`#?56oQ}^yyFa_Ba-amAwuG7$hP$V;4Vk|-@@>lL zs*zWrK3sGsH^^|p>+!#k%QL%$PzNG4_C--JUD^(=b@w!mc1ZlBk4jQ~Zb{#oj$p@a0626O8 zo(DA<^(9B4i{>Q+e}ZJpzm!kklfPrj%cg4^y09w9{XJee?su?6&$g}6W~t*@y_eBA zG7_7Gh+F=y3tDY;FuD<}ntVV@)1~hr_p_+UQgHkn5Q>mk6UChp zw00JTOeANu53o!XT9QnQsQ^t~Sh~7>jqSsSvEPA3ig!=aC_8jiTaaylGIY0hFxFVo zyAgJXFqat;a<~)ABmheelFP#a67w=U8(IlrEZl11ZHdutHj++4bGAZ%0zPJX{T@$T zi)dyA+4TvyNd@_;Tpmf!tdJF>EVI*dvrao2=uVoHX0}dR+ohqy+*B-5x-s|O>;70m z83)M>JLXOfEcZ@+P970!+iasB9BT|AM@J&}FS5sAk>M*wmaBT0KdP#?>by}mB+-1c zf%03(*t!3WusXK{ReVh#Cs95qRr(-|v8N01SyAl`$K=QDBoltXv4uu(Q`&BsH%aqfc@FDWd28iTF!h){&Qi1i3&3@WVAPo27jno zz?^r>>?q|1|8V7;r9`m2*b{WE?A&^G7Cs(wj5h}qU7v6F zWokV-)pT*6$^Q7v|6c|cuR9izTIw*KCdHS)t;6r)Be+-T{O1dVmuyNv$gd*zy;mRI z%sPOpxEs9oCZv<+I7ciT&IRhk!bYUJ_VZ=^v{aTVbAa$hsCnbM=U?X(;A(!kx$S+1m-1ufvPiGWt<~gu?%iF6EVWdc=}X(Y zCKYMsL~gQ#9Am$OYBDF(_tz|bUwW*T#LIfgQX1V>b{i3B+8|fHfD`28GWn`KSN1wN zS-FWne_n3L!iPo2DJFcjSZibMdg&r?P)*INa*F2H_V|I5f%`)Q$$%igbsdwmzmoD3G1{D( zJcA$0YyZs0r$cg@6XsAeV#H4@A=AhgabMy3zNuta;Z(j253za%`I2Ebm2Xf;MsxQR zy8h7KJwXGsPByK(scgrmcb3MNrzHx~V(%POgJy$r&4&n0t{|N}o4;26@o zGxE{f=jAC`Di~D1W$CHr?rc0jw@J`!=#pLyb!>p|ja7|zjPe?sCDDtW6UyEm^0d{b zdelFt-u`9y^`c))kiRDjTxq()g4*91cF#?>`nY^!-^`Zb@YA$>mLN!!6RXsq(^>cA zpWO;{P`9xm+eQ1p^%j~QA$?7V)FKy)l(v-&82ONYqJFNLZ;8@4a2Jws+~T2UxP%cbdp#rlU1}^*`)0 zUn%$4)+hTo{CILeyK~4|6@yH^?iQul$K9pVN_OQ<{^L_u4*l)>fMYIffh1kx0u8R} z&HuoT_lElx^i^>)g}vCXUi_yQ90Lq`&Edg+oE87{TF$#ip8IDV{(SeouSI&73^bU1 zHvON@6vD~`+6CTbDxCW3!v1vwWAI}oACDv3rqr; qpbM13KaKdum;3SS|J&)>l+HV5v=et26-ESpZpbNLE4gYK^nU>Q_Fn1$ literal 0 HcmV?d00001 diff --git a/img/mamba#12.png b/img/mamba#12.png new file mode 100644 index 0000000000000000000000000000000000000000..afebd048c20a6d54ce5f9a918e5166297d6cb5ec GIT binary patch literal 87075 zcmeFZWmH_*wg!r85(rL$1utBJdkF4@yA}j3+?@mx+%;HmcXxt&aCZsrZm-hkb~l~V zcf8+kyj!Dc)ULgBFPU@AZ+^4B$jgd7M|p(;1qJn7Li{}t3JOjS3JQ7y2?27aUMdg- z`JrbjEG#b}EKDMA3pO^jfZU>va0us<@>Rg<_PX#@5PFXN9HSb}M0LTz8v>-d3oT~@9P zmch{HE({NwQ^}Lgq&Y#J;?54B^)_}=g4qGupqft&&+sLkr&f(Vq0#H)9Z7IeVWeO+ zV;zhuYG_>vO;-WesLVN|7Drl)OS8dpMj3YBnlN7YUfEz^hoE(Cj|HcXAQJK@gO}Bt zSz)(wBv~s{2WJCi>XgG8+AZNfEV2$x;Y}l^+Z!ex!jI7r3a$};#TJ0(pEH7g@#PIE zWXM%OY7)jWGEg*-Ya}S>08=P9$Q3l?^9u5Tf`X0oheCvWV?aLd(_sEgh0{xe{qq`n z<5xx@MPUgE$hV@QEeK>~_W^8A_e=}Y0`!!rlA67mj5N0)*pflt2y6giaJIDmRRoIH znHzFx39{EGakjLuvg3BH{|-)ZAMa(KeE`H^O35_$dd?zZ9ydL3``76r2Hr( zBqY4HM#kL0_oDw(9P*2g^n<;t*N!WDcFkSSH1cMUf45|1_dkyX zc|gWrDU8evOpJfk4Jpd|>n^vvsWZqz{k^FrWOyKL@W16`icKLvRie?9t>q4>`@|8W;G(EKR8jDJlU zKgxK%Xfza*Ae6*=Ath(%y<~WI0_CYLxy1S`nT3UgrcX3k@+eu7pH>eTA3Tpxj#h#5 zQmCI?f8gXcKwfUIt`F>LoLCwX2CX7vGH<`$v9sE3?RXhE4Slt(G+QdMhgg2b{zrEi=)95*q{%g^2!Ej`GXiTHl!jyk6o9n&8B?v?QZ*L(=L3rtK z=n7X166k+!D;WHIb{aceHzaI3( z|Cb&#UfTYJQGcwAWE|jiDG_!kJ?Zr+E|QvuPd+s_x5mjvKjTcfh;AN>BzS8efmL4S zV=2+&ZOX&iir4Em1kV16C}(vk=|iPVzb}lB-h%cAyw^G}gm`q4sCCo7jhm0iJ08w% zvxSuqQb@)|4rwmM=)gK~c5Z=>oRh`9T&CV_+y|`wn5!00lcX-D#dIiK93U(DE4RlNcF9#P2Rnqg+hW z9ZD!8E4?+_8HBq589A6NMFRr^Zu4WELeKAA8tP}4sa`!dRRv8Z!L59<7QP<5<*zm& z-BP#Q+hhFa_z4B#2EYYlxWCzdcP5dZTz$_wAt2T-lNZ|V}*+Sqn zqcQwbGL(ReTrPzd$jQ#0p{%}PBJ-$ug#1yf-dU5)jt5zTq zGkF^B*4c<{(T8`cVs?5)_n(``r-vLJ2R=CGgo136k<+5|s5X<1Ke(f_b9T8ZtP&W!ztyLpavtyU69a5yJC(NA3#07(oxgb*W@r|!Bux+hb>by zM8mZSGDo8;v$D3!t)8IgQH5~js{Y6A=ceCp97Y>Kv_Ab!{{VK|&wRUS5IIN0VaSsxVBg=kyS6Jg>tsnD zJTu%1-3R9Azyg%(D+vS~rDN%INVbNP?Fvvl@^mUDXiH_M#c-1i^grc%0O5+#TxylPv0qRMarm5=R~Uzrf&-n{6~@eg^08rt^Y;imYE zij6&`JHzVUADAourtMl+u1FQ=!RN4(S*Tf|p;xpag)Wt28)F6(Cd^?qk=;GrJxce} z30nDTHD3FL8^>*ed1gJfMB_Y>=0P5RaA#n9%r&)Sx8uyKcl8snMTpFYXT{gW?kz4g zbs2Ft)YO_4$feqI;Z=dmu-(F5PZCG!@=d-{U7&&Wds>w+>V77&3v-zB$vjP6l&+4S zr>H2Zc6ZXAUsJJ_wd-s*M(^91FB}%)%e5kZ6QP16kd+0E`C&s!L%-5$PG{EP=Lc** zQpz`Pgqp96YCW$yLur%?dg(n7a!JZ&?DwK`bE&iLK2S*E(1&k($tk1l51zx4o{+}V z86@s4RP-0XAXur$f3yeMm>rQI4qlSlXtMGTou{X2w(cuADnJbcsf+Ju1=E*VXUgttsZiY3j#Hy5H8_?0XYu$GwpM`%& ztJ^^qh>(A-GsE{K4`JNh`F+NB&!rbrqeuPns#c{!+%&z3XSD9enue`I|;rzMn?fD#91v= z0ri)|^-m^Dh!ZS4=~vj{&7*ZGL#ASh7u?Rq`@V!GH}qv58!AC96YRWcDpEOG7Q&9@ zDK*}9IV~F9s3+2Z_6u-g@&h6ilLy5Zpu{3_*=2hxq8Fr4k!EKV*2CEP+y(B_6bS3i6i|kXHQr1;B!-^st|MpDF#LX6 zA|Z!+m+#FC?ZnJNt<$GkyRb#Fk3-7OdfPKnYigGaD;W-Zd%1niRbebXV~cI?wd`nn zcg4KF>Bn)|l?;18LfrHiyWW>9w#v|%7f|=U2uTQDO-@>R#Nqnp^4>|ezbr${zS_mk zcpDL@7PqX=Ts7mB#`G>#KLi^8a_oc4&$66Kc&I}9(k0!tT2H*|5S7T*1$dOtbN=&h zI)fVM-lOCr>Z{SUYODq)W+9Q+`E7s;A*Nirww58S@h{qhC9;~nKyZ-5Mj~=$% z5-vj{&OBdq6#gtLMnB)l|L4*-_*0|q3(I;owE#hASP`6PdGs@fC?HBJ2thJFBu9}3 z$zD|8!No928;YgZHUem*dEa<;sM=0>paD98n6!i&FLPbHmb`ZI(`q%8trDt< z{4@!%m^x#8k!{~=H*{_M90s)H;kRu{ezld!{QZb3n2Y>T3EB^l#_)I*ysRCK935Xh zN25y65Px=pO=}AOY}K%pV#7x)WdsLyuG-><6?&IQx6b5}12t9fNoTas03NS%EWIkf zGW;_w?cfu5Q2N9bP(eTxnaK<7XDHOXOy5aYj=gwG z44o`gti+J)Ee`$cUQ6^@VG2}?s5|w=)cKI;@`}GewRl8QMop%T!hEiJaOXx{ zUwuB__JRV-9(|ZoQoeFI-UXq7-&w@RY{z>SbN%1=?Yavf= zHCNw>wG#QeHSNJhZCd@ln3CMJ)Xa=Tdt7D|Zx+dw6@W7}$1Rl{ z9+lV}VXS{hl35Uvnng?H|ozydajU(g8<+55^UVp6x-*Z4Gul7dvIluhJc8cCq zlJ1L}TG`*V4n74$>pbH&S8W%>_uH56{j%15o@C$QdhD)ecPJd$DuP7J`}N&RKaT78 zZTN~@PIz4h##GyW(|Y3pe>mm?d4N!_6UZcT!?S+hxMYTe~mteyiO_{Pw8L@h8ne0x8)4#K6%e^HoSS7lY0p#V3x z$~9=0D*I!D&x^Esl7=ohCXy9-ezm{M##Y3Z5382@S;vYY9GgAN= zUT(n?kB8yMau?K!bOz$asn8sijw$WdBA_$j(#cZO(lwCy4_u!jvF{iEEZeft5U@Y; z4!!sD$dZhUkkzgS1eKkPR;YprKTqX78~fefCQ*aalGAO=Uyy2|EJoELO#6r&Ljncp zAPPd5i|4LgYHgd%f<<$yIomEVfobPgD_&l~s(2JM7_V##|40j7X09p`@I_Q$tw2Bd z-Un6DE{NX4!RAwd3{&;0?QF<;#ygT^1bm`T?|uB;0oGQd(?@9USuKVBGh=_^j8+<8 z2lgSX%dz+KHmpHvL?2Y(HA>8xtICZD%s44^u{M)pB%_T%lm`BQKM0o^g=8_>+|BWD z@^;17SQW{KO)HXH2T;Pwb;pyNEgkSK!=EM$jV@@wIbtDowkvis_dB=sW?Uq~l`HGkp1HXe-YXdo;Yw)R`g`<_J zS6eO=7IFuotQrZf=od1{QsW}lvg+L|K>PXc;NdcGykKpQmXbcy%Cl7_G|ARDf92|k z$Gdj}Igu{-_yue?FZ)zO%#E9L`-44v!0TSUzFthKvuWlB8k$&CqDf}j7g&m)yP=)X zkyc~1#>#@m6;*`dG4NHe_51+!`pmE**lO=047D^^rmMnnAd4bRg=qPTw3OZ;SSeR+ z2FQ{%iBHA87!Alpv~fRZKGY|iLJVZ5p0~DYgXx!8U1RKThqr?tQ(n4l&G3yE=|P*6 z58qGT299vXu$dO9GK?t#4c)d^KT)bP-q*>_qc~f%(!cP@TB)xwK1a!_4VsjcHS&W` zh$49yX!E;um`}PTc9pIMFVV;!2>n3%G?}XE`_5tPCAKU~uS91x(urF>Yf@3YTPqmFX}i%3HJHGm#xUmbF9>-c!z^3v zK4ZYfc7f%lP#pF9P7z0X-`RoVM8ygi?%=I+DSwvnzK@7c)Upo_4!XQ#YvOnH zZ*VpSdP0WTh8F{u1E`F<0XxI|uDY7ASU|KB=bq|e(RFo%xfmMNh-O%Xen}9UH@-Rg zI?e$m64NV`-Wb{%D>P=I#?B>g2Usl`Eg}EGJgh9k{Z`9f>X+|+Ea_GGK$ZUJ1JM;# z=y6p@98l1iTfb<2=>yp%|8cM}T}|Yr*z_>6EnUL!6Q9KhY3q3?u@d(mE1& zE2QlL%rWX`mT`L1!ZW7ctqh!YHMQ73H%3Tnt=0>=~E1pw+6b+O|kQ~5T^T`i8 z|F5%|E7HdacmN;aP(CO)=zaYS_|9)sIsVh{qW=%%wOxk9Ks)>$GLe-q@^LBF@e&Z> z-tQLCcj+5})>mp(TLB|6K3fr&E zu*P?{*pp*VU(4;De~-FUyZvt?{XgzsLyt<9v&rp3ykU~QXum)o zbiY{?NhL?b7u^|zBhjJ>{;|6_TH?=q%i|x+gw(~wy&bodDfHdH{?=qTCC0VN?20G) zb04C}L`Il^eu&F_Jo8By6>&{$F;&Dz4`Tv?4*+xN2DVb@tnlbn0() zHwP0yk+tMc(Dd3sB=W}-)RQ#dba$aReFfMJSq%zGO3}xo&akCQcXb%pL%Znu(F;c5{e0Ph-lF!y3Ptm%SSin zC7F2^8Z1=gLbO&dSDQ_ySO8;@J2MuEI(E>f{{AR7j|P7rn4^V0ISFOM8}-#LV4 z=Z(FLbeZVGJ{eyNlSuWvUmpf2ZcOQVHt&eOfJY{0W>yF#7BCJ$vk4%flKVO$b&4jL z@YHy;vQxB}=-2KpAu{uM-FszY2%lP@ zOruno8Q{1xE>ZIa0uBg)iKknp=m`%b;Y-v+b>x|~a0Zg-A!x5%zacw~YMwDbH~XGV zB$h$vgJbWFgZX+-#CO~7h0EQ_*{WS2wBe5xl)nY179pH(u{ZL^^B}%yt4@=zCi0R6 z*bonu2^?_)0ng5Xt=;>C)WfBk;gn7dMxDmmYAn-i<70qQZ7IZ&(W@Wx7@$D5aT~F; zU6Z4-NlR<3>+752>E{nDi`*4^C{E@9o$n~0Jn(?~ztDnJlGwm+~vyf!8t zn;@3DU<;A!GXL;S(f*e1od9i0XCF50dX?$}8LZW7@|x$b=pg1=Y!EJEUp2kvy1=8W zdbP!jMa=iUs7Gf}2&SDr#%`g;$XKvk8y-BKM{9i*AXxah0;&ymA)Z)l_Z%!13d8}p zoO3%fXjIA{G#z@HPBr^Q%NyT1izf(xQ&8q_eOn{D6wRjc$i5%cO6^P*P-RKQMV!*8 ze1~jXP}|+L3W-*<%jv#Tmis4^NRHG3`pg)5ORiAAKqiqb z>f;sMOsQuw@53~i?OptC$m4Q)O@ZEkUsh!~LMvB|{Wb<$*Chl#+gIUpTCgjz-`;T@ znQ!q*<$dV+KFMEkGEuW6@yji>8J6J{4zcAEv;naXL}BtN8QD<~2sS!^$WsXdPG(>M zfJ)uaT%N171MwF%ji-~E{9$~3UYGZ1`Ct2Y?S}T$PgW3%(+0Ai)9oAjCH3$7A*wgL zerrMg=)9@kb__QLIqd90)gI=#bvM@a9!JZ!ZbQ&&O;vBODWj>bRrH=tUm14pH$L9 zdAazQh7ve0lCM(qhc`0hHAf$-93Ma|jHqmuoz`@_1+Z54ejv8dMf=jHOD1V|LE?)9 zu((lb+ZQe)-UmBVLLhDE)?IbzQ0DbiKVwvv)Qs&0UqMZ^`J|(IZ(8WaPU1;4SpZt^ z_34&_Zr|+e#Xz^^lQXJMp#;fKA!kozHUAO85& z7WRblb26|=QUu%pTKq;KdA!V$u~g$!0#qZ2fcUkzHd_pT>TyuO!mN=BGOXv)Ocj>V zGMS3**juPkSk096C-z;G>{)%kF|2%)O3wA>| zjCZwavBG3XvLuDC@>{pkGW|$ljlCknh!nL1|KpB%!7Q4g>^ z*yAK&ncx!ZYsQqrz7asbD>#*UFtLG1#paorYRQiJ`@L@4lgux%4l$UpUSVQPLqEoS zk8r5v1vdwDGowEZ9j7u~gznGn*wMm6X|~tPXch*z^BPU%DGXLmRn@!9$@WID1}t*A z|17Qiuv2eEtkjp9lj6xO;u*!XItQi$0Y-VluF*0?)hm+pTj~NbH<~}%#%ooHd!8?35zf-RjIiW@Rc;IC0>`!hHeUAt)l1A3_*k1LZ5Rz^G5 z5&4Yh)XOu275geGdmI)@;sS_0#*1`@Pp(nd<8(b!R};PhYMKC%w(C6%NZz}d?rnnB zp$|u6)6!g&!?N%@1cMOrfb_L_qUr|ftjO1F0VSdV@2JFS*y;A!Z>}MURx|q7#?g$^ z%%YqIm*g%;Cm(4uaM-H{4H4YFfv4xgQQU%g6EJ44FQl7gs;Yjeme=ZvNZdd|%wJ0Wk!|(JX~|ZxeEQV27(8Dp zJ8Zo!a}6xj3tm@65$J(X33!9-lAfT$Bte)+^)Ouk;~;^7(R}ygoFT#W1g2};ngH_J zl(&VNn(B@AkyX_R?#o?vOR#7=l;Gy9nxyv4iN^;q%~GWTC80%Evg_wm#zVEc2X3JC z&)}@B^94``N3RNM?$2wJ)F0am4RutzT(|{ms$D`5QkOCSnFfP%6EsC)DFREI@X1{4cYT9x9DwWGonW% zFD`QmNmxw=M(zXHg851-ZE$bSw^FNa`;RUc?dEj9eKhxv(+mEww6-EQ=ff=~plD?) zzloQ?Y7PO!ibLL-kIUD+3VquP6t5c<@<~i|ziM-{YDj-u)^F*DILEOuCdWUD^1WtB zh5@mY-r()ZBD4L7rR~l-pkQEZm_KLUh(=>Bg|@Wb8;CeH+HlxNVcNv#GHJ5pZ*b}b zZU)H#FUwwzR;^V*eA+!LWRuO3efKB#Xad#Me5z@1KIiy!=(zM+%Z?A}gm%0zGmmJC zF4_=+1@`sv?aO+e+My+9WgFL1<@*F4H-FiHAfl6%(}%f7MNLI0UAx$5R$E`@3mLcZFh}Q4GV4K{gz~z~JXV!cSs@$!D)u z1L=L4v1oOqZ_bJJ&aE#75>Af%iUrf+jqz;!toG+r54*qXuXV}TOXC#?daIzLA+PQ_ z|CIK;L+#+2op*fRUNz!L*mQMQ!HqM*HlZ|yc4^>NWR6gY;gLfzVc@1~?4b1F!O#s~1nKnK2^lYBXyF`SCB4hw91 zj?814hOrcC+Er!?vweZ5Xk~SyuDVq;;-JoXRJq(Men)0#=Sw%Z<*KdTpRQY03lmkF zzzfTTa)}TduxNpi%hQD+ElN^|f=8`OV*F($aNKzaJO9YXj5o`q+`%fP-? z6a=A9FW*n=B8w)=T7nlVe*l}~?~#zqiCiOaAjbMog*qV`=54l~w2*qeZPi`jYG`XB zEJ)}6utf&jRHOOyBFXA{B^Icq?(2;YQ$7nXSufaMnNSh4D=JYkv8v_zu5saoaHT4Codgfb*Sq)#p)W`PB=n+GZL`S;FMFl_|mW!X+990v5U2~ zW+rUou;qcKYZQKxz*_U&k@45>pPw77Q(>3ljY70;RA!t!l-r_Rn)JtW0=VmBET$;R zP$$J^5-`c3)aNlHI2>isYlBg1un$JIRE-rkvY|-o52pr_p>T!fczd<|X>2A?f6Q0| z%#7DqEOJ`Va=jy<*w;(=Vs8;7_hQ;L%9uV=LNTtLaBFxSxtF03+WaH-QzQRlvfv0d zlYFkTo$xzB?F{#0EnjIfXs6_!=v?g+8vqDez`@ z-i+EPjdQE}X5Chn>-j6IGI!qHGEHdb1F1uH;}mArO3n#|Hz7FEJWdu)1zoJNLo31u zWmH37hg=m)5LXG#8y9&TcgU|YDU+ikCyAmbSq}sHRtKr|O3?yKN}BJq(4pp$u<+vP zk$e*x_fC#YQs+1n3#SUViB!9c;yIj()vIEC8W|^wlIG$5RA>wUmbPkV zJO)A)+2d7A3rxj8rWW9BS&) z6t~r<)OYc=kcV#s@ub-(RpwcnY4Un}fNiH5I$S-zW}0jG5gmA`(}PR+u;qs(|*TyCpYhCNfP*G#wcLw0S7DxX%DhXYyklv~K6;r7)uA zWquYVo58E#BF!STo!nSy!!gkK1uPcl$cYRf-NNDwoGiI`)nca#;|?sh;9(PtOAMT~ z858?z-_W+@jxAMtY-lohHCI|ps^u#ccy=1EP82=n;aWd%lXg90QW+w`o4p1i` zyhk8^aJwxD9*)pMFaUiEqF2T?I^%T86Wtycob2bZY1{Ncr`gdsnkh4S`v{+CGpoxz zy=_Z19O4>CN6`g3kycj)ZIFDjMq3{|1{k_G62?pj#+oP)cA6{gwqEpWP*qzO8rq%j zOgKpeNC8DDGVX^VPKx!RB&3wS#92kIp|`e=r=U<7mD zTh1-H&sAGht;!3&JQ}cL>{+;6G{-(1txC#(=~9gF26zt&^<0L3Bd=&sYpl-osm-u5xMpW<5CQaBg@`VMoV8gExbaKHJz>UKT4G`a6 zN$c)SHJC_aMs+veotzGDM068PZDC{iJ-Tm$`Ym7V%iswN^@UE0h(co)!HAQ2vk*=4 zb)L1QUhwrB@Q__nLk4vtWMe-t;5uy=>riwbK4c#{`=fO2;$Md+kmIL{QHP%-ryI_YA z8ASo2adak!xnjPX3bW*0XrlvPk#dhqo%^(tRhCUOI}~@i_Y=clnWzjYo&~p z8TWnM7aXsqua-09GF_jmS|IV}Hr=H@=5riO6anLzd$!Yu`j$%}^Fx?;%!9nwwqTud z)(=)u58jUo(?V0H{P9p~NX(O+>)+mml#N>*#3yey^p$JFBP_vaZ!4f_exkNnTK(h9 z#>|#(WpThaksyG1G1^`Og=Vx@P11O1faWKDb{Lz`c2JMF0Cr_P1aCK1WYoQ1+MPTP zg@A>xbbZqr;uU&((C6&OrQ+ypP^Ec(>2dZVq*o9$peh8r197$%)G!f1+)YUfo@e2< z$h>S`myfwCqHEt{aUI}nJJ*N95rFvY*ptf5RlEFV5b__XLWvdo(hMcr_UjqJbM@-r zq%&TZlO4|z^`=qDVR5a@_;;T`J*5p*qt7pc4qI+DUJy-#%bo&w;jI%=B;+_u&GX?b zfgD(qjM-rV!LLB@KRTWP`c;as2gd!-a-=jp(ag=Kl>&K<4LVHb$1wkWM`2$CSpYnA z)SI3Ln_b750^=H&+mI;`1Xbt0D(=18J7pA4ZN9JUgwpES>wfLxs}F5BJ7m$X&T7)P zCdUtQK((@z=O*H_RT})MNZ#pU3_2@|oIPfmiVmYWI}rSuQF6DnQm1D*%-q50P&5V=$6ExM{k}F{OJ}lo&WL;qR*=B_gG8rMVU9)&*l!ykh8vf zJ)V~j#YK2Q%-0)*awXG+WWEN+9g2rHLqRqQvqkfTEHv@wJE0ASHHG;Q-A4zH+VwL` zaRJgWWVLhVXiJklLHv7s;qKfhlG}XRP$=UiBjwdv@ZcEv9}DleK!G#kWgI-0LgQN+^~ zaj>eioEoBMKpXOd_Hfm_b}zbPGRsWLLOmht>JQ#Nuu)h}8ErB*sc9XRH(vIH{Zw-A zN_4!>!EZP|eH*iQ8`<=tBGi+#YvSR)HAwlUi-@ltiAc7h9nSrc1rVhEWeI`@r}7+iqt<8y9^0A3Lc4X zWCyWoX65R$Oam$*zW39e2Ppb-Etdy8pu6cZE3{i$@?zje3UULPWZO$sdh2 z!fmfLGoCSDm%UGb*h#y~W!7>`z5$8+P2gCNRGfxGg2w~H564tZ*WNZi#D&1+Hs^;% z7Z&?)TBOVwQ!hOgbpnba2X2+s2)sk|DHwIVnN1&?5`%7i00ZglHC=nDrBtj>u}qrb$zCG5VS8z0m#Q zsLY&?nrm0V=gZ}--f24?<{!yU@7}S@EjGFKy~2i@wj*#xKU~mw~W$1eIp&(p? zd5UqAcbF`B(+146W#W1hcJsAkz&hY!#nmX&9zPCdb6sM~&9B!)mvYwF>XZNt%h~iL zRj4Uoxbf-PAR*|No7RRgY^H7dL#N4(c=IcGSRhw4V|u0M@!ng5PbAYuS&}j5-?>&f zS`FIc^eD*(hWv=Pf^g}hSxq8i7#925X)?yKF|Q4GprO%-7mXl@Z7gVze?xMtS{XGN zdy*CDRZz{j`BdSg6xhKz8}Zj67p6&DWsqgGRDlfSVJ~f?6aCI-b({zfn_6Ce6?1hV z8HSPXv$lmDic8FtAqBJMDx+X4WvWf4rvC*@( zTvoDseRYf@@>aX?|8`chVETl^t(+5-mT5=~2a8!7zLVFb(dEsveTu;S6|zQBfq@hR zgTR#bu?=yO$;BM!Y&-USr<5_@r)kC~yX^1&>B6;VdovS`O)^DkYhG89cCisM0f#JY zJEzmCg3>pZPNL8P3Z~yyIc!3>$JeSoV#Rvn+ZhrEls<1?E^{SPY7Iy~L0uv4cXR_7 z{V&j%tIXwG|Joo160J|Tj4IzA-zOYqu& z!tVhkfAs5P1SB9&|Ko7>Q^W2ratf&*BsIe3E0^TI_KWEkt0een=I!6^!N1unf=0gx zEi9p|ViNxx>i={NT@(_}_wglv66N2s`gbFrpFvXJciC)G|7*Y6#l2DDnc7Xp{u;oi z7ZE~`)E>?u0*Zg_7v8V9N6{;b#3xwr-z$*?36hH6osjX=#r_wo2I>DM`R^HqETI3J zZ8ix40{hL(NF3a!aw*TDwLR1hzHuAFKKU>~M{-rfQE7u4a z8(b~?WE__Te7pgAwwnV0F5C6!pL;VDG&FsWL`+nFY-hnlWC-I(;+=b=Rx<&Wjg5^d zgBKeP6sEXzx*9;9N%UlWkap%SGvy; z+P@@_&(TY9@y~InTfcrf+`jo# zDDfg0iJQZ0NHyQVe>h%+j!F8}D7qw!y!V=p`@vqqXC+R&4XNwEn2nz+ezqqa$MUNt znom1MzKrYhY|t03AGiMXb!S4tA2K}`&YH1{k88A}may&QQ(v4o&key7^V*0WW?Q9| zWD|8Ew0yG77!2u9u@y`$)!HHfz~IgVwF_&~O03fTOElTA&EIlgaFRUy9ds0ZcK&+9 z=4*g+Kd`BzI*MeO;nN?Yki_og=Ulgnz*`s!>9+WH+?!IH+!%Y${~<(cF~!rpd&hpl ze+ezhvD5!zN!4Z{hV_Tw_JO!lMh^z9g?x9hVGe+QDUm;(hxJ!E=04mp(_zb~p=m~c zxIcI?;Vww;=bO|4dQ21cN*lmCBoL(elf>ASdr5!?%( zC)vE=?%oK(2Ma{U2mJXYX$X{dglHH3Pjaon1;$Rt&TLwvx!qn)TlZ;PMi`}u>|em!_$d3zk84?>3f5c(!)r zw{Mw{FyG?kMze+DTnU99srHt~+rV+Kyzb#bt-jyg(fwsP=+qd(O{<3xkA%%8@+rJ% zXyY-dWS#STdVe(waf)jJV?85CAu;H{!uJwfj_cN$&l1+2YUrMCaPN$v4Sn}-UK>r- zQsBptQ|pc9e>DBk&Q>w{BSS39Yl(mI&5cz@4a-JJ3;cluEW#q?FDPI%%bcT%U!Fm? zneKAYBQB;!CaKzXYr$!E-|e7YadmBNBWCJ>3St37s#h5I9UFFMJ$gf96cn04;+yFn zJ+CQ#QJWg?O1z}bci7fHyHewEI0lzjC#3B{LVru#ua2o)@5jOfVpWPXrMhL4&n`Y9 zuR(moOXA-}L)Lq99^O!l3wMSR&TV9%1@!MI7uXY>9{L~>kAfhywY*tQngW%ga1LEB zZ469IhuFt!urHlO)rrZV>VEB4A!yUl?-Y$ie;$4Fk6Q}LL5WeD-G3qmEy zPJ)DVY7*cbqqfy-`$Q%1h?p#`J}BY4|XKk_C>1)FYhq8DPvy}k^EgEuEY$DtPZp^Tn{Ca#B+G*M z$rhxWZqC`zSSM>1ofpcX>Z|043kH6{8ugw3e*8l~HtN@NX$cg*hGs^=Zb8xRjaDhZ zbz3l>ESyY|hy42nC{Q~7u7dZnJ*ba9&xcR5ek$F4rragy?&>?5?e-{zwp>Ue8-+>n zR~pAb^6T4@mcn|(7q5(plCV3zB`|x`m%qJM))PhZVS~gHZGq0vmin`0lI%4g-p*NW z#MhMoc2HK4oNvI|mE4FQjx%Cl{c3Wf^1Em@a~>#g&f+=!)K9&l6E zRQj1v5`|>6xWZ*SJnA~cIRCLN5`+_jL$A%C_}%|Sgv@J>owKZdRmls2nXK$URoDdf zeeixP9K|4nl4n2rextt9x);dTRlPsgPwaJ9JT};Zyw+_#MO_g>5lvS*x;`iFQJXTaHJOd>_IJBNUUyI%TR{ji zZ_Vb_aFh8h6I+y-i2reP_#Zv#oeIbD38rhfIU6ts5vQ55$=svzg9c~$iR`n*O8Bm~ z`>_QtvP1nX8ZqqPcjb7mqaj`&&0*v5q6S(f`cYOtECzyt2AhLzzYguZO&fN{b0e$X|fuOmW>tFq+NFJvmnXIV7f3=#IFVl}=L)JZwlB%>WrpYuB!1e&loo@UWwzR!3KfS@A`U35iqqk>J6RO-OPsH? ze}ad0#f*NfRIF7uIP6{sb5Yetq$o-2M)%qr@qgK}@0atdyrCgdZ!x{P z=pugZ6fN#w9FC+5+d{(eX`hcNDgLWifsrGjp{%Z*J9TN(jLrVAarQAAFbZL^%SLR{ zr<;^4y2g^h&9Rl~4A6;BKe&uKYmcw9&WYRiZ51VtkJ2jTQp+T=V^izk&}-=pGOQ~h z97xW5EAO2w(){XZ68LtXRJrWio_rr(ho3|kN$kMaYt+iymsspmjvpAQu!$jbW5G#E z>z(xpFT&RpcD}&isp2Vl_7an$qe73#^2@&$we>kB(ihaxhx=PPk-#hF!M9kn~~AU70!Z;?$>sypLFjQb_qcmtI0fE<(vKS#&(Rm6Rs;J3YcbN7#A-4GV-)CvhgYOK7ZCu`E%b=ailbEn(FrelaF|1FXMVi< z?`@6eN15%WP-`~9Kni}1o(y8ZWK>Y3Q3I82H}wzd$u>kq>9k6$wLciYu5fM9+LmDixtYyc z%oYNgkXpe=QifASb@yq&cEDx-oRS91%!&5SIG8yYmJTmc92$d0HBRekwdm~VU01FM z+)l_DxB+xgzj7o^psre}Dss6uN(ppcHc>ie!m#-jXT21Qiivp?GTZ*5`^%8czbwSk zy4aY{l$2U6wkt#t@T6JjKK;7(goIqSbswblB9gVUEj3z)Tdg&B^JmaCWo==|_SoN* z^71u}!;hoqn5O>|4)~p;ZFF!o4{r1XA-4LlpKx=s&(u0;WoDHW4T6^nzm01y;gZ>| zweyE?1O3jayarZDs^EKk<4~3#hYp!+wy9Rb=y|4#9}u*iU~=&PVx=$0?RMST4=?k7 zAJz=i3V8|F1e`;-wwk9-sw53lMP6{-HeCS)4qn7y31OneEa)A-Tdt9!AkB569=MFJvU|bxW%VNfB!Kt5rJ&&$UaojBmM{B z8RU6*yQAJxj`;9=Hu&sCL1B@Mnpy$bn25;iqPWU$*ywS4ExK#wM7LN#3%qpGu&#v% z9yMY3v^LQF0~MYDYGshbhpK-R_ofR`qs}5|#s2Osa4%}?T42)yuSjmb8y|30%gm_9 zGG2vyN5kN?JgP5C_s|W;ZJhfJ-iU3dG>Zh`=yew20lk?mKeAJ30mNf)Ho4AhqJX6| zTEOd?GTLjUTT3q2SAw@e<<~g=UR^&RSt{|pkO;&1&yKYo#?H0~CRtEhJ7Z@oRSEZt zISruFsm3i=?I`k&|M}ufjjtu@zB(lakP#rGkPI%zZ{As)s<)?$m33bLX;#130a=~t zGyMtokD1qTV38LGxKWa=wK$G=n?B+cdAExWm!Rlrskn52LX_21sffULNwQb7Q7V%L zCXC%x zVJO#lB>twKS`4d{aZ$I7RNe`N||cYkRXedC?K*OT7k~oum|IqXEJpKi;!JE8o=mV5uX-c~MeaTpsjI>!w9MM^FgIeZQks z&o13{>i}yfdY$60Ud*Y3lgMJWl(HpJ;t@Dr{8iJuThKD8*wl|4zck0>kk)zSUH}n%%+S)>uVcIqL;J*&pzmeKvxF_e!&HytkgBy zEEa%f5i}2;CynGJz@86I9nGp76k%YS9O756df4<147pfFTux|)^gwZujGeD?w&4DshRj7 z>)`?^Kw5kwT~$Dy8S(G?B{1;K>2MYjgv6Y)2luZ7;*rEtJMD|@F8d`QXhJ}z%rfnj zw-J2sU!X2LcyoVjugj|(Be{Y6@oj&YD8dWCk4w7|JbXKt~IkN3@ard)q= z4I(;)+a*~W{U6uw@JC;tFROraeM)SZF{J3n;1^=i~BtDsP!=j#kajP>s zbYjK4PqtO>R5g`_NEKrN!2h7|B;KK3r=JMSd)jApVM3~8WG&c}O?n)d*EQEJzb#m| zoNE6#C^v`Qk;v-(_PUs zQDk$?>m7UoN!x6+7!_+Rb)mvbaHLa*EM!t3;_XTVaC=KprxjNV5oYIR_DxsamYrO$?=Wh|H8Oe}vf0}o0gKery@ z9s~LkugFdJ0BH2}#ioxk>|{buPDfwlod+dYw7fWkQ#EQ7F-jH68#}4|;~0Zrr8PBM za@N>3;Yf1+E&uh1y+JZ*rc@FZd2<1c97&}5Iph=Jq%C)DyE>;2NU;&U&grwHH^e0G zuQ5)Rs{bR5zzC1a3240^)I~3!g6L^mm)i82EpbL3ek5a-IO%R5i zU2!h?NF{fE z4tASWPFk_l&3sSq}E3k^E~ z&2Un{DypHje<$E_V?S(rtmrmXt`dJ`lt!(R0yyLGH3D8$qx8CLdT{FBm<%X|p3;>C zx491o2~Ev?SwOl0!QABtp2y{#`tQ4uPHqk(TOKxK2o8IZl=>1l-L?XC#dMn`HbP1= z_b&^>cVa2CV&|-vhvfv0McL_Zx3|u){~AqNPCK`R--QGI2XAera!f*QugJyvadHG> zcIalv>&re7^e)Hg@O@X0_+3+4oD=QF5nlTH$#(dUoUvV95?nZyTOFIb=@X#o$@xsJePI^)fwp-IE@ zOdX@72uAaQdh{Uy$!jtR?1%c+ee@Zi#ze!q{Xqp|%b@N*F>SLueAgxH2-ZqGbuNkT z8@K9#@2ia+s-II1GB2v#0seE5JC+RY_DpRg!Bnxxit9ubT1gvzZ0^KA#EbMEDi4Ec z1ZI`WW{iKo5OTtqI2et|E3aIxAa#;el1Te!9ayDi);>v#KsqE^-eX*YB9Bh$7|9R1 z++7hr`YA%43d;*Fj0etFY81ZgI8lVvqO*&fHjgG2ykgPn3RmIL;#h6S7}VY~r~#^) zOs|)voiK#SXlt#eg(W%A?+SY5_rz5C`qXdl>fe1#|3l=%Fi*aI((@5fHV zJRf%?Wkx&OPFF}@f$jR2hqPM#lUiymLr|1NJy9~iJ{}tD8Sz&8Wb)*q4idF*J40%awwlwz zJ}^XatU8oC`*gj$yBqU7#07*qi3mT>0OV=JNRF>KA%n-Hp;mvPmih>ea2!5aoG#ph zXT^f~C=+F+_oAoLGwA0d&p^R)a)*PiD+N70t;Sf8H)dw7 z>*1VammgAf&s?Ou48Ja8b>igsma3Q5Rw&N(#RMF3q@XlYYzCodC-qR{%%$m1n#~Fg z$iQi?eK$Fk3Qw^kRBQw?Ze#?QbJz&qw$KORru<1Fd)|oNGU%cPJ4YdHB=xr&AwfF& zQ{KRBeZYH<;;;VpK8j5e+G8#7-WWFRLa*`cPP@rxzNnC3!hP&EFX(C~N$bh738Sy6 z`_99C9hsb*BGp2PZX(<9!N%wvYTH5+vo}*?n>tBVSgEg5pBaC(+T zc|aN2U6aZ+zWd8$iAAcNIV4Ai2X2Mv{M`)0e{OZ#&MSVtPtB{F-N^f{n+toY4W*Hb zBYhaCLo+Z5(CRIW8?3u`dWqpX`Y&(b!s}PtoVjoDYpBv1z87Lt$m!ID)Cb+z56RUEK!1ZW)D4Nw;bubUnf*C1&}r`|Q1*v>+|p zc?2bXNO0rikjq%8N(lPe@Fh~odxA}=Y5y*BztN1#$|Gx3yslHU+xgp52=}?y&qm%S zV+z5pGZxYdLK3A(qGIo~=UjD)JBEPDO(!sQXP8}uVJQjAy;LOE|v zZC!6qtie^_!TOrDHd(8~Y{HRMgdm$iS@+BnyoYPs_{h3u=!daIw*iNt4I8l`hAy3c z@pg>g5$1XH!gMS}H~rc4K(58hLkB8u(=>p?;@c#V9yF_6Gj1yN@I^4emRy^>u+T6} zGP&ou!x2Dt$A61VZvmpn9r4!(Q^Sj6c}!)c1iW;ViK~79JQFYqY{N(pPiF@uOKcDR z$>SufMy?KTVqU(o%YZdS)RZ~j3If3>L7p1t&1F$IgfXh@5IURHBauFg z9v5isWD?P0*Xo$BaF`&QLQ^UO3b}ud_^UbP&!l)T{MeL^ZE^ z4D9N>hq7+^c{$5KSBAU|`yQ#ej^rJ33O>K*qy;9DKA;`=Z93;F-ize;tZ$Q3VRIgU z?H5U_3d-Atwo+y`2mC#ji49fvtl#kQGK}Cg)$3FVtal?gakAo;QuwX_P>BT*N}X9% z0VL+K`jj)4$*QCgn>lq+nnO02{sS$4^gtKx$cKt@lg9mVhT41@9N_#^7k#qf_g1mi z*!&BRj9B52PI3)U;`wh=hL5vhk!nAN1H4935x3Mf*^45Y;P>h zrX#Sc#dm=*W_vK|5DPalacXy_6pe;r-o1Mi5gv?)l8A|110K-OiO018H>p z{j@-_<*zzWVK~+y^CxUJ<+j5u_da9z6xhjUic7k9jmA>nVLCDYmwzhKKs<;nLFDr- zr=aTt+rS9aLG>w{`h%RBKpy6tYw{o@y1YQrRu z%!G5-n^W~_;*ly-abaHUh<^3R9CU9%15K+wWon(G2yg5jPG;Ib6e{nWcE8c68rkju zU?=NYT&I4r6oue?PZPqMFBUbaG+=09<(UK1^A^t=nKBxUYZZ2ch_`ujd?l`bcshYJ zxym_Lw5L8_-WfZbnCjuLNy@_bv;t@9B{==ZhtCm6tmKh|Uxd>$whfS#WJMawUB7h7 zX4w^Ha1ojwNZx+=R!OI%`URddQX`R=a~`F9(6D?;5wDJ+&xb4q0ii-kXaZ#rRq1}s zb(&rkDfbUH-zA>q@`3Ihut&Bq%`@QEMPu3W{=gyFMyDXD+^kh^F|9kH8{oSJKHZf+ zmV2`N>0KR&hX4a~pFQ6Ue*V42KVSC=>gi?X8rN@vQU*KU^sz+&q;}^C|7#r3e`qDi zR}mhkNcVecz6eeXMSmXy1-M$;b=b#6GzOT5!+iqlm`-GX zipH`|W<-CoN#hX^WXg0h%hpUwK^L(H?*{XMhX0oQilw}tZXkz>@KzrRKde|pgkME= zxF0q1=y|fNxR08p`5yf)l)|cPr&yhzQsYVyB<(8V*pt0kFI_9?)!fVE(7kbQSF*)| z@-U*qDo?N$HNPXH;$P`k`6MtEuS2uq$Z7q+I^fS7lz=hM<&?q5qeAi;u~7RkF})hH0t4xA3ocC!VGt>GY`m~)4i@#Iz3!CktrdE6gx0@o|NhY6T@NnEgQpzP7D+fK0 zn_MYNhl$6C<2#b!t3}2B=S&iF!u;_^#|Kp;{_|}pOl=yXBC} zrp%q=|Gk#~zsh(mDnMM;PW?92``@(A|NK%dnIAf{{zyW33OQVl>_5zAL1Y)0k7rud zx8?u)$dn9m5OlD>(DZm5?Em+pfB#@t>>HgJ{2Hcz{)PW3{r~q1XfXgu?5Z)A`9IPI zAa>XQct&h7I_dvSH2hD92-v$&0Fu&Y#L=Qu&-#^X4^k`p$PMEZH4DpBe;7rCYvpgD-`lPV^sX6FAq@Jr|~ zL?x(73gBz1Jcj>H4G>gE=<7pIF#H-myh(Zy2PrLz>rmH~Sc;_zJIr4bp$&_JDa?_m zD5f)NO^gqGN|@T6MpHB$(d2Tt4$a_p;;wix+a|O7f;Ksl%qvOaj-dtAW}b6?j1&!I zyKZtOGL`yWF3guXSspB%aGNib)a(q$tIbvE(E?hP#4xl0DSrp*!Wcm&_&!YP&wzA9 zqTvuoqJ|?5rhy)dp-iX4xkNmhV`_S>hB?MmRXRP2&X>pXv2>n6OIDe5_V_14V0eOZ z2{E89LQB8_pp^=(wmHjG@ii0-G_92z59T;(yW@YC?myqGcn~9mg(-qxOU za>2S!4(i@Wti@b=(2wI834UCD?KX1^25of!$|4<%B1{Z`Lps!+EmKL(XmJ<`zBwXS zAh0~{1oR5VJ1(Sul{O??dsbwYD51fZdeq%X zxB@f+Q&+*K4(DA^#jDb!ejiYL^Nj8XvkNo_0ntw3d?cUZI9f@j9Pdp%@AIGhs9Bjb zP9Z{{olW$oHTQkKgVkcWpFhYvUk@lRmuMkQD_Fx}^}b;{`4ct&JSy&G-q?s$ds5f) z=sIM`$HVW1##yN(_>;TTyU%}`|9n%OkIvY?-&a;x8^8skGi)q1E~J&r(2>nUJB__B zs!_B$2tmcd!n!yWUypv3aM&3m0XEGEl!RWQy7&81k76s2PDcfUPC7UCsod7BQI*qx z9ii4_)b!zS`K-WnJZnJp#IZA)fc4uus6I~wU~Gh#AFAX7F5|_1^ux#Y@A2Jn zq(v3@DX(ekCeOq0pEkNvm>?5t8eI}94F=CThAswvg*-{8425lNc;=!C$g$r3V@R^y?4HB#&U}Co z3-=BuOJ82@umqyIfLOBFo4AEWw~sB5`SF(}~v^3Nuzmygjaa zW<%hK;?_4k&Dt|TxJhyz{QlREp4AQ9uanMIFy3*cv)U4>Q#VzEegy0`11wwVEyDWg zdKNF`I~R&?qYE-OUldF147}Jpu{xG(0(OStBpmjKXn>V^JRXP5AL|;GxEqTv7J;KB z=Maj_Y`xIO_cB^PQrN9qk#EAUex*>ykKh=WFJaBd!pBOav6;)uG@p>e7u2KaJD5#S z984FuZ>lt@PG)yB+iy!c+=v9%ZC=Imi@e*P^<&^LUm9DQt2}yAWU{rPj-gEUSbz)u#+A(v}Xjl0yl8+aFE^4uw*OaK4i}YJaXIh|3tG{;;0;lTD%(K4o3HZvi*%_Uw=0Iy=c zP-?%p81m#@@H$I`NU+?Hs6q>wIl5psm@@u5azz)Ul;3*1cvaP4Jv7Pa6qVX1fofR= z9jL6^`a@#vXhp&K^`&X{6Kk%yR+IPJgEC$$;=>KUTOh!4*t)mJxV>np&}|sW)pMo& z@aP$O|5(8dW%p6B$Z`k1}r=k;RSL<@-!(u;$ ze#C>D-5`U%idGmL!_9E@c0*#EpXNv6Jg>@cyk44SdgCVj%(Y>V3M=@`g+s*So5V0%oYcpHFP2YO= zA&hpuN{cq0@&%OMO=>t3EpYU3*Y;!XbVyw=uUoNjbZDwj$`~wZ^&oM^A4jcZ`e1*J z^l5vF=iQ}hv98x&X0<}2PP0fR)vCc_8q|>hh~74SMXoSOVKO>2`>}0nUkp05xInVl zY=27%1k&l&%eQ@_RLwPUWg3nn5BGM(xf-2dJCb#LPF+C#1iALKUq-gyw2J%!(~5nLnY&>w`jC-v|iV|Kr| z1k*$k!-sGLMcD;WnZfUVVr5CDO}oYMNY5V7jkpn9AJ1yhsDdh9qgP_}ym1+U!-x-# z#2f7!R)YQrr@(Jk)J9xfTJA}b1uoqgS?`pZGSne6%`o1A+-GSov3!d|w;?rC^Q3W^ z;N(Zo?`8V}j>TRBJ9s+7g}fTwlc8kS+YWcs@$6n$YrTGE3q8dUo0@x8ZI3s`fD|L` zE`iKZ1R_Ud^vB_hS-9%*B;A7h<+iG>~6NNr`LMkPYN9=Up)Ay0N{p|B|#sChaW87)<9%2 zktj0}tty=~UEO-{nWB+E#H)nGOqmJh4Z0x$p{dz~R*qL<9(+4~9pKz} zytrFc$x|Qlw&%M!qa3mQmS}7IU^Jm+>{cfGiSp+^&j4fvm0HdpzW*!BiBr7Ybwa@B zo}nJ{p-0K`dctUDFY+jk6L?q%c+N>CG0?BSFi9e^&An^G2_?0~pve(f7SFS^ZEg}I zs^)1rQ%}F7@%yz9)17V3*w1})Xuj#QoK(r;a(av<`Y`GFNoHNaUJ{CeMpq^4pN)cG zGwQhl2fG9+?OCGh?|#ZG@hLMT_dR>Y+Z#%n^-+<;r>`w7Fa>Hz5^D~$8%Zn5`SLfh zI|r#wG{)4yFe}PDQqV2Cu|jcW{5l%j6f!sCwtoJwm3obK@t!+F1}}n|V8gFC4=+uR zSQ5wC+)lXWYqf=^hew-wlEC}Q53dI zuh^gnTj!B8tz4>f#GLC8ilSduUf}O)7pzjQFpWxmj)-gX-lgmiz8a{({<&Iiw6+X| zO*t&-&x{OFc@nCeWD*Ve10uRzGxOJm5Pw)i zi31jsNqeE9?E#`{_wht()tMV_?A`u zyJ$*~RMJ7ZLnw!-&uZ8-o=wTc=y!e0g`juKd)au_Mz8dYX+kl|(7AW{4yt)hM-s+?Th#}K&(F4tLUwO*g%K!#3=c;id z1Y13oWmo&wNOcPmK$~8{FE#+^a8EBx+KDe(Z07QU>Q&bl2V#n?yK;nBJocn(AYi0} z{mE-Im11@-+OjNyiSU1yow{*?V<|MLg{@q7<0am8;&?KiTVhJkA4IX!kpoI68Ys7X zcKqC_QQkpE7%P9466Px0lPyn2gG?WGAM9k8DNcQAcnMvJE<(L4?Fq_BHC_|}^5Myj#~Ag7Nsl#ztKsg!<`av;3_hv5 zVQ}L0C6~riTH5b*`Mq^#g~3~~f-_?+l9`p+P_?~80EQ$DE>)UVF1ZtSj^J_};7k{5 zEbBQJN!0CWdiuq@`8J-;76NJ5 z>;Akja1U5b=9e#WjQX(Zhm&bTFwmlJ1^}Mv(>7ipDm;Y@-j~UFe1AmU=2xLGD$u8p zc8ms<``$5}C_>Ed;g+r5(|QiqJLWG-4Q6MEvwFnZx#O{D#Xs=ImiP6-2uWMD!N;!o z-nWZY^}3S8ek7FL*BW)Ox!*YF)&fEr2IJTMY*kvfebQ8ZGQXzsqOTSZ9Ryk=)VC0w z19DW0jO}mi+JAp#%+=mSk+C7rAQvlUkDr$E811&7-FaMtr7=Hw82yp9f&7AH zkx+AkIUgN&cX#U#4%#?tRNtt!5TPy?9n#Ebip$oRre{kPOu7fh}&SqU~79Lq(Xp?W#|bEJG{QzGSI94DL~dHh^2tg^nH8~fcuM+?CJ9yrhMp1}dR zZQH&>-TEy09cWNU*QpoO?UbcYW%AAdWd(McGdz_wF6oj0JLDBVE zehBtJBn1;o&)_31OpxqMtg1OKnH{%*i6ZP zBq?wc&LW@~8YUV|9`<{A+8NoU1=^R)SDlZN=nuOgf!>^MM<<4l$yU?Dz@)kYuk<;0 z`|aSH%QOD;4K_PLfz$_SY#QZ)(AQa}t2%tj#k&2;lZy@LGXhtmjbNtFIT)Lt~B@Q&$lp0B?I zh)v#yb(Y)V_`E#y^a_0?m#5bN3p*H(7c5t(LSu@S8|Zn_`;^6i4Ev=dfl+SPBC8!{ zprb!%*-EaZ09h+~6_|6=B-wn7Kjqi$e0EXe{pa33VSGEky7gT(;48pb`%;>yHorX> zU2i5F(rCT2*}IQ4)@p#g4mwMH@{}Y0PQ0_fIcv0)$kd^moi|2_%A*3+!Qed7?!KBC zua$L{zZch4{xytiK&#&>B5{RcYlW|FwL&qAiKd>^62o%Qd+)a9#{IV*f}q@)p4z6= z1W{7fN*e`#DI!g7ab90Ux;BBVSANNa(GT+1N!_S7qA|h`SCT-VRG~pU1tS@&Ocie% zCW8jUO(K@RmuzwjA%;QZz(CpLHxCg)IuAq{t-u8AfI-67Xk3x+#ms>4}Z|6Fp)pKrFdqY$G;WCbyaD$;>tC77ZU_C`Lx0guT*hMPHbjhEpO z|M?y4*1)j_>kW%A&tMcpMvsw_XI$iEWeZdR!m^+9CLO9*!g0 zQ|$aYP)ONazNzsQPyaT&2z(c@DHij|dcEfI6N$?dkM*oeel3Me+Helk%Z(?nWoorY z_pUo7$v@}1ocXtQyitgdpzQG8hGVK6PrV&l8Fj6J-1BR($M7F=!qoPe=qYC7ZSF+U zy_%2${%sO!x#jDW=vnY#tii(lE+X^p+vwLxBb}D5hmQ?7&l3VsQ%H?1R||yi zVx@ocekV2i@L`w~tt6d($8|XFOF6OW%JZSIH}jVlgI4?Pca|tfjpEhtwir9CzMHuO zNYlS=mz(uW&U54%sQh@pAp1L?ES0X#Z|ZDS+F-vtKPO-(h{u$_X*619-?!1Vz%h(U z9^neWXM#ndS=o8oJI3Vy=MZt}5PyyzbK0aJws5u3PTxEFAv`fcc%I$l<*$RUwZ zoU7~YaQE{UR6Z7Q-=uEZo4D-HRhYWxbLGl~KYw`N@dAa%@08i(-47W-&L|y}c+h+W&RPqu<76t`6CSKw5B*@tQ#z+ut6Sy2~d$8Q;DI zQ3DSK@MPpLSD!HOT(HyG zO=FgDBbi(b3F?_H4}LEeJWu`V{Fx0mOPS;aKiIjwO3ULtQ`IuYw(CDh#plE>wA}-! zwyQ2Z)V`k-@VGw`;-}p~2_l7jG->N=kv93M>-8wP&v2&E zwwX*DLC)j|4)gd~yX5Tr^7uu_U(k$|Tr)Nl6I$AUe2TaV@k16ha>zL%HtI%CIgIJ0 z$DbfOS6VGlMg-1Xx=I)n5|>=|0w}rsR0iYSzV%-Bx*K0ojT|sUU?&~oRUA*c6^zQ! z?@GY`L%d7{^yj=hr=0(IzU5Mz%lomyed8ckI0~OHE221}jxxc^ha_{5%Vx1%V*kCM z>>KTDHuJ=ioUu@A*CBtzb`-tpj~p+114P+vUyeA;ql?`f_l7Wc!{X&`V58{*Ezs2- zG>Ly&E9MclYw|xf7t1!{@u*t;C@d>qD01Ii)M_H+yEc1-OZ>2u^72*$;2i0}>ru}_ z9@7c5dP1+-7a`_^MbvRyt+hPGjgooL^oiAqa=dvS2TX;W2jPgqk;L_afy_x%KCH78 zMcR)?lyxI%{4Fjj6CRYm3LMAuyItqhBZb+vNe~?8)$&gHxOl-uT_W$2Dz6TaPsYFb zq4}!g$py7_gQ(=#J$*J{qmkR)ua65~J3VunfN|DRawqFsH1^AD$7pA1H%-7vKq5ZY z_mE%H$~E_p+%+4HVb16O(8GN~w3bh6z)*xLOUUb*)A4H(9>N%oIl>r}ARl>VLgxPU z6C<{8VoW`UZFkG3kxt@gI~52PW5{tut~u$D}b$ zUxe}f$=o`s#`6^ys-KzIa{siM`@-}Dyb%uEAZ$OHQ{u{n4z+nt=xej2OGy-#;Yvn0 zf3ZI!mYA!Glwvk}jDz7*fOcA>#1ffZa7RUt!#g7WdAK-fdRW(Imp=dT1KqRu4RQ$y z49r~F4bP!LJt(WG3LYjJ8e235+Rd^2JnqMc(?U`*?BIt=r_)K@TH|Yv%^)UxY?@EG z>obR@Yk?v%+1y|E6T5HA4wHGT0^a!2h;EvfTnED8k%d<%gDSjzhmyDgoRdFIY!gdE zG2h@!7sdwP9FDkSG{TBl@uF&gD>nV$)(GIg2#3a{4x%sj^vWNYzZ3z=zt0XIbCgej zMiPS7fdD+=eQ4GlLSg@33hO?ML{3kFA=Wo1t5QMLH*-ket(TVC_2+CB$aL1u%Hnv6 zr&M^I7u>KH?^cC^yol!!32%?TGNB#%F>&+0!A@th_s{V-?~djczahTWEsO%Pk*JH0zyBB?OJQC8WA zabL3fhzQ&k^2si|C%bFtbHj`63ogs?C>A$Qo`VGrD}tmgtx&&u(9guNimcBl4RwoZ znijun8+!NFDdiG7RkBh~cw8*)DcGzzKzg<}H-cf8Lw5$ctY*^Fn_oke$urRgqs?u> zbQ*Qk(L+{CBB-+mlI=Rm#5TfqLER!CyVyK#X^fMp0vSqy$XB6#@8MYT_*d$(H_-oe z0SH)pQs1d;8lx*_6nca4cAjbzcRd-t;Rk!*DQ@rj;t-SaTTx*hZ0~k+^{OZ@kYFZJDORSc1;YvcKrSrR;IMVJoP_ewm-3*TFc|B%dB2hdGYko zH(!KiYQn+9H;&RBpz?xNWPQ<0ON8_D^E6G;2Vzm}u=!a-NTh z9ntN9M5u~Pa=sCo@l`LnI&AE_(THg8(Wp9gVhHRS^!Yc@8)l{M0jt3uyV*XP;a+eb zkVB*3NxrTRZTqj_zrI2(u>w~eNj08GKht&i&g{8ew+Nqu_hyh0as2_33Jy}0j8>Bt z7Z4LI23l@G@B-x`M(ba^M%Z>6_VSU?cp9yL1_i)nh0LZh+u^!HQ|)@~oEvnI3J>iu zYUXvCT|9{F@}cp+Lz?M;ykI@P4RnZpeG8{C{G*1}n~7R4H|`cDfFs3?Ts%hb?s;6T zx3V*!Ux6P*WHjyxW@u5$;tyXSMz3ysr zONm`SH+pM5{v`1cJh8IN(o+@bsCj~u-*g~Nw|qBbxZvI zEgVEs5<}X7X$$3g7328YWSlyvyIQIV_Dy~h5HnBmx(ceZn3F$#KmiR(9DH(`Ux#x~6XVaD|87c~=mMxFMO zz#3202M5QDY#-c8q6JLG7ZG64aVEpl%5Dcpv)Qf5QH;y^y5t&jOY3 zKgRiEtCWf<_kJY(a1xVv&KP+svzyUU;}h1W))NVt9+agI&=vJxPcZS2a!5qSzWwtk z;Z}h9Ztz?J{W@v|mki3a(R#U6WxrV%9MA1^bdY={d(Vq|MjHtJi1S#8=|)1=tJUTD zVZ+f1=HoORf7`h+l|pU7+FTRlP2F%>)V+Ii7Ltq>839C@oc%@GjZXjg&wPt*pm-Wg z8$5|n5NQMs#z@BC&k=F)JU**G$S(xG{IS{|XMQ4&5cVX4`(Hp2=R4m-Q!=IRp_jO4 z)Wk<&5iCZ>50x-yuP=}7&wj#_B1i1j_?jwJ3cUu4bi{sqFwUVgdLLe;Bkr&rC^t=P z@4?7$nzHKdzENN(Yq{J}lDPf(Za!i8 zUqFUf8)2ozQEhOVXz^Bbj_y23$OvtC<05Qlc$wz8K-mjL;F+b0Mp|@~fxOj+vCzYt zyijZ+iDlGTs+1fe8G$O9Kz&F;`LiJ2@8a$NJXuUDl1YYb(+FmXmjA<7L`?m>ag7qDK?M2E*PaR! ztU<5K8O&fqhkz6cqW1N#^4xexa6@L$7qEkD!whd{davq!5fnF)AFT?#ah+?yL0Y*N zIK0h0Me9+m+3Ih&_YJB2MuU&ArR|Cvn0$_`H^!BErQYNmDS6zi`2765^Q4fUSS-&& zYo2HRG`~0A$dZ4kQ7+f{${jtbM2=r?y(%e?;b%E+q4EQqJ7>K#@+aH-_e1dcM)1Tc z&{}vS^H9wg{3qroP>YOz&Gx3Du=><_jd%2Dz>&Ff-v;qAQst6vbMlf?(*;t|1nCUk zulrlgUPq0}Rf^P;ytsR;CzG+^K}OSB!`;jnI#xB6+)smoivq39i3NA}W6tgmVP&!~ z;psIIqS1+hpi@O%g*{f&<=M@bi%cm8V?|P~b4Q#-h!VBiIi25GKHh{#=j5HA-U!dJ z%g)`S!}sT_knCl#`H^nDa`*u+^-8gW<9www7`IP6wrm2_BC<+$nB5zsHu!{f3?659KCYCG@*OB z;@gvopUw-IX^ow7_cFmqc#p#5?9V>Bc@7IRC7O%e_`(u8vbb||eD$$?eaCr6G-vX}ES<_LL0Qc1<9PDahxGLx%uZ6cfH^gh>m{II zzt+oPFtxA78278%JbW&WUB~^wv!O%B$b^hBDy&(HeNlcfAJ3UHQ)PtwQM|2-CL0iud)VBedw5x(;_^7HX!R3gNUdJ{lRYTj za5#JOmd+uonMIjMGJz=;UuF2>*fvGG!E1G%q=meXqS-OsE+v$ck_;r1O%l+RPWCw( z4aJDV>$5by4!3~H) z_x9Hc`KkLg983-=E0eY}K^n>Ppp;)T%M$v6kZjFw{!{#Tr)6gZL_LBy;KZVb4D#VMiWh)VIlu>Yotq0qJ#dy}Y%th#eT zAVr?ChHteXc8N}1&#~e|^8?P`O8i4hO0DT=5$yo%9pp2c^#!CR^wGyb_QhK0;4bHb zfxkwb$$YLTJl4lC;;sGBWWDFaA$X+JuDBGqMuGVd;978`RRMiw&J&y86rebh|KslF zR2f^*;(8J1AVYpC7OHKReHP>WNpBVB>q_kEH(!Yasv6}AMV-WH>B`k=2Ay`|qq&NN zc*>}wx+W}rRXQ!YU0M~DQy*rPcYAfUh7YK_vX)QKetB9Qsya7f0^w0XV(-pZd78=t z-8Ro2@$Hj?`2L}rV7_jsz%yY-S5?s*RM_MoX1aZDIqQqL7Lf`S8~a{1Q+pC*pUJsZ z1%rfPr7YMlfhAFr7n3IyoG4>*5bSpObV;;xnK*hlKeHc(AHXfQZ(TyATqPtzP~@*i zeldD7-`n}aX21m1l@!6XFO>Xu$5zXD3nk;M4(g80xhL;P`k&{eO>Xpqhmp{9kH_;Q z)X-D06{rQ)|CE;nOAwmBeo9d<<25L6Bxun0XYKlE_5Oy|D8GnaWvek;rQ(VFs4r(g zwcNaX&o{ADu{hS0etGE6S?BLrGK*R39OYH~$9XVMlP#iTUx!Y6!fT<$kzRU*H|KvH zbh+gEf$;>T@k9BRa^&6Mwm4#0D+#+XJ@Q2rlMN|}@(J>JBNemRg{Gj%V!H#LM# z??Q14abT&@3yzkVs7LwrZhStM54Mrgk4ox|p@`s^It@Bgkygfb(9QZt;t5(Gs~^qT z(;pCTyR-CHIuZfpJ}uwQvbAp)}NOCK)@41CVrjzGlpmTI(G%h3*CJ!q^mlaAdU>0lT8 zwQ{LigGy8GqaDKE{9i27faDFzJD)1TlJhJdm*cn<2~>&2zA&<=_i_?>wXaX4R@>bo zTvzM%T<7yP4`z#<-20=mmTG!5$M8l33!T{@mVjFDP?Hd;A4lRF?-UrtxQM&_0BOKR77TI?%B)DhYJ9s##0Wi13Lv4i#6^}rU2jU< z)xOzhF`sM(GRwGQSj9RsStUYxr1~z0o$9i0G34pQpU`gq{1&0#n<>$ebGBOk9HLxW ze_`a~o{I$f7nAj?MiqQbVx=Man!(BMAVyC7ayNy6Q_8D{5c&mD#=re8)8Lnf|ZK~E?+ zAlLBqK)t_@Un`q=@HpjaR^u&uaNL)?LuvzWDTp3#t_u4K>DXb{lH-Zsh+xO{yZyP$ zYNVG(Br2(x9AMCEErCq|YQ{$@qnL~_kCABzhqK8vO1hmk!QF#vfZ!I~o#5{NHJjacv-|Bm=iYxf&Ght~s+Q`i=l2MQ$7f~h zuHFVtY3DX*BM@O4H5x3JUgq~BtYCsMOWya8xTpQaQ!4Ul)GHgkPyzVO$Px$APt4ya zL87Bp@QQ;C>6C0VYZ8`#Os1B+Ji|hxZF7UuVJVizD%GW>{;~1_*2{V666R<#9D2JSC&tb*|<<&)sP;~*sflQ1xo51I~YLRkym%_s4dw7_nAPvu$@ zG!I`c`^~7?^H+ub4RqrfyrgeJ6Qfzx?L4NDuq}>vv4_&f`e6{UDguEjBo?HmT`$$JHWQ^#2TNutVEP z_d0sZuzwC&hB@7JZhkiYP$Kc;eKd#8H{%t_!X1A^>DgxUJh@MaeB$(}vDk5ua(Du{ z{crV`Sn}j&oWA`YGxQtHI*1h#?Eu0Lj}G)!L1VVRUpf$^=ecq>W4Ajg>ye2aM={Ve z!=enic#AIWQZ48%^f7pX<}uO_?t-h$+wHRTac|VaCEq^~m5@JH>~yoIB$>mZaa)NT zQTQ{_)%rl3cvExp#@@`>jWY_q?O&#czn=wQ`*pB)pmlpTM3gOU5I_sc8)^Sk;*h)B z@w%AO_3M`1pdI#}?@p=xq>+f8mw39t$_OFgugmlLpqRxE z-0e)3k~N?IJa2TxWBRKq_za;b3HmiJbg%Bmygs04J@EdE(k-I654$36r2BqvOq!5b06?MZ(NKo&(^xI z14VxK>i-{&Kj;9W4Z%S!2+&H_0>E`y|)~LX}X$4%L zCM(s@HL^b*Oi;x(!{^RsHz%@pdz{(-^ux501x~l74gpH|AC!m z-&v6s3}*N-PUHI3#O`81QZj{=ODyq-Y+&*Cd$>L7*W>~o4L$wfe|>u^EyKVd8ONO}*XMTH+g^_DB)>wTic7AukwcFMc6Jpo%}$&E(vP z-erU@-{c{NXUs-_JnYlx!d*CLuA6 z$?&noPBc*C>om1wfF(55^bP@K*aO-pm}q#L&agei2Pxxa-&bO5haz> z>45iS$;-`Pep<)iyin!aK{qVv5jR>nw%%QwYACpC{DJh$x8v1qNy_gs7a|{cG7yQd zTkS3{EkHuy@Rc>;(|hujvc{${;((m00!&FzI3URFf6_8r|9C&sWSiw+u{3!OQGXFe z8p+@VIFPxeVSxz*T#Ipl96l`|>y$E;=WBw1>Kz(o!UaB$WGe*J`2|3$mu_-C&ek~J zQ7lw8El_I-)#CKxAMxOE*rD~fMID(V6!3h_Ne+H71u53`yo?G0H4X_jQ<^mn6&5o( z^qIa~U}XsLH>%$TthS=pASy^1weQ$aFs}y|{5Uoas_qz#JQT)`Tf^gKs*E&bxm$13 z9Ja<;JG;BZ74s@KYFuU>>b%B@8STa+G~996?6N}v zSct!3xb-DDoklgfGad|i+p=CIfG_yoVU>L5hynZ}B^*@tHzMMSU#G=WrS1bSm(xXYzejyEG*}!`!~ZSRr9!&P zKvC_~F)OFhZ)1uNr4*4OP}%u@!i*eMetF>2Onk5k@M-P#0aw*~Anoq#uG#GpZ!#HH zz0xSgXfRoteB-+>+(dnqaWiNUsC;12X-bBt)UWT&ut*K4>wdHYsLv9GQ^h%r(s9() z$d9i6d$ZL_x$>#+QC1l7na`vF8%F z@M}OZfsa5FXg%X@g3I6#`-BCs%2E19i2Yy)NY8OuV`m+A`b6TEjIKw?n8mesgIM9o zK^uU^Ne<_vG-mSy-i-|v-VqNt z_nV~7AT*J9I_*M%F_WN?EV2+R$uN!T^{Q_o*&>bXJ_p0kSLDB#hB&RC6<}1_-p>;8 z$R`iUA0d`%N2WGq;8w4{r)V-6#)iAZ4*bXwi!T3rgv7E6DCh`}QSaDOXc0TH%Eo@R zs5#@rw*u~yJL3ZX4lyw6N&xe@Oo9D*AGr{Q5Y7=|*U& zWx)puF57kE)2w}mr-#DnHj6=|m~?_U!)v!L&OpWQOOuVTPWU4xb4!hEt@fKkl2Saj zYYpgM9%h;~`+(qd#!yKNiasD_?VQ_x)7cpsoXSVkrehNZ@T6&gLXQ1bx64X=-!Nd@ z*Q%jFqfj1+)i$t+U`bqz$M31g(wD-YdV1i>NtzpN?9b^4XK2$llJ8A5d;-{o)A^QAOCqX{IXpf}S~V0R%BXN7NA^oeD8rHj!N@)@dgAq;9hSjJV@Ix?D@>{|cI?QrHp;B{&a`f}VogvHRwhkFdeb9LT_ zvw9EC%{X>9oyx7oT!j!K@$&I)Sd|&ls*ja8Nx$ln|3awo)&urp_&{6ntAZz8%e=B4 zhjWkT)sfF4)|Vz3VZ|1lMauKtalm_&pd8IeG1EXYiQ6!GWB=sat@e6blzu#Rzhzui z?)!RXyVEY&db^EEegf#McYS_vnQ**xdmV8`gDJ#7SkL(W;{LSI#h$>IW>pRLs$i5Y zC_R1#aq<2?7mM6cx{R&g@V`M>i zDi5Huh*MUZFl*)~wcb{q23aJdv^s^Qo<4bcY2-S}!07Hs;Iss6OSs*c46#SHd~fD~@fr^r(nuz*^?k#9LZJWh|j zFCyQJ{rX7bHziV5cH;{f?Fu;I-dk;fao-nx{)f{Q3p?bjQxF-#UG&sUa34;xvnYL; zhd{dTJ3|X#UTz+-R<)kK$Pi6;$8%4z-@Gr^%C|wg_8d3ypyMtuQx{TUvRi!gvs%az z%6h|{AUkLlm&yTnj=fchct8EIcE&msU?`(|rOlSDx|54A492}%|5~W%;dTvH%i_Wb z zV16F^K(`GAkT!yJne(&1fpp1bJ{Jpz_3ape0)e^ik?_k{dqN*N5{P5IIP6V;x{bX~ zRiNHj<4KHMV3vajd6<6Gg7U++=MsIIQ^$xenvcW@NjL)bY+s+u7DX~$;@LuTcQ?24 zb7GqzJ;@;ttwv8-9U4;hfGZ>9_LtG8;1+|^FU-4KVZ4xUbZxQpC|g7q*_NaJF`N@s zB8b*KgR;gp9>!Gq0q+9d>z5|k>!~l{1D#H?yHquz(g%XOqRRxgVh|!uZDEKik|#L;-)I@NAXij zqES7-uE~{&QY;hsm^WT9W^xQuZv)T`$h|BVi)+70LkSD1S@f+-`eH*vD!c4VG?o!X z47LgRK0cxadyFCEC)9ShbxK) ztb$<0Gp^8Wi)Vh}*MA`jbx=O=*t%M&q@q|sbQA#sWo9o@u6#W~H7^!F9JP7UwYlfE zgIzD(>0g~~%}8&GUGI(g*uD8Ub9a=5$ieL@S}XK^Tu;Znl&~Ewem#M|S7cwoqC>@RBJcj*{4PG`sqw@yqxJ-X&+d-Jf)+r?5H2dd9=4 z)t%VRA6PBMIb}p}PIsMyOVBZ#0FIHH1yQXm?V%d8?g)`~ATlZ0H_z|Jg}iaDW^|kR@}XOD z<>g4?z0vd{zhE=`3zQKYCj+fOBDaUR^H&7JUc3y0hk6SW?CN*oUa`c`v>9dN1W1ra z>m-$^^3kUOm|31;X)uK>L{9e;wbjUm$wIKDfLd10+Zggb7ScF=0%30%N)=}UTGz9! zO1nYsnFhxYAUaVzbou1_Ci`#%u$s#lUK$>j|4FA+43iO^Bei{s+Rv#F2M6Zusj9N* zMNe(b1~Q&3YR6xMU)P0udjsNl5USQj`s%TpfW7*abFYRwj z#wYUwQnVl&h@YkCYFZzOSMG=!{NFg+s(j|oP8sO1@#g%gE%fpY38v!7>9kIDel(T6 z(E^#&c_uS@C$^ftN`T8<=aOTkX3P=RwF387OZ{pg9@3xt^Ba7rK{dcKLI{y@h)VYk z>h8zK4!=fl#$u!nOpU|w`=od8D_Kn^=lh!NRyz??&^9wKJ}|l%4veSNE?&>z7P8JH z_hRT@FpW{gkV|e*tH5L;BGZyTu!D{GF;b3>mhSyZeG7R(#zzhIo8k0p3twH3=!T{V zDeA5#3s=!Q0{}xosnd8mM|mCi6m zYqp>=Q!2FTb&&U^lG#(SbdT!&p4nfG5HIjrFS>K%At9xK(^k!NEJQrKcog(=MWO4JzS&Ym4Au)@+@;b~o|4fy z&RT%i))&{WTC6xXbQ`$N&3Sm*KudCLZ_8~cw~<^%jG zzhh^=@IX-%9|<&F8)`}OJX=a0{Lf<{$j4-?YkG5$7@Q7g{+(x`ID04{-JOY2919-y ze?VTppm3k5s3&+AhmzbK-Y`!@|AmgQNdNt8%Ti8NCy;?N?Ss$n&NS@^jwcC3Rfcn?i6MUimH6{IZ{{|I46(J^Q z`cGf~+3VkXhi-%WRD}4cX!GCImA7$EMF>|z7x3>--v73(M2NbIUIok7I7erHc;(@o zUVMCZcJNKp?y!xH&&%}NwkjJdLBgLR%71meJx(wiiSdmyQ)bBhjBcuWdiDpi`ljVj zg`f(G7hI=cF!;&Epx*AflGpW&(hj9B>-QSOpWA{j$Uy~=GKw3=ka}dS}aC=q|L6EaZT4J(lWz|l|W|Hwd)llFj-M8 z?=As+O21kz2~`>n)1wmbMy7f_(s{U@$0l=F!!}y?DHj-QC3Jir8>~uFdXy}GAi1iqj)uh%P@!$UK=qi?9s9K z#Xv&0YuNn;cJuA(N2C(RX?_5C7;(Tg&`js#2V$~F3dW}G=AdU1ahE6|NyP8-zbCvU z5toV&^}J!bzFZ>H_-euoAoxC&Z6BP0)TCSK5Mbpb9xoG3=9m7YR9o!~BLN^`jIEZK zT)70jj@Hp`=etBOgOBej>_6^NpuxcMDryx%zex@J(>?M}s25TXYpN7nJ-gKGCO*8> z=rpX?Y#K>s4s^Sz-arvdTlDUM%HDX`mADCtn3PVVCK`{+zIUd|*x#?!ki&6RE+{|>WH1#THn8^HHS#(9~ z9W4+prn;iUmu(aE+V0_wD~a2sa&Nk2Q11~E21%-;OO{l>OUCYQnhDTER{$1EU1mxx z!eRx=^`D-iu&7iTLX}GuRjjmHf!K^|soFsCH{CF4u|<}R z7wCYafjXmCnkQFaIjMY_*2iv#Sisq#KMa>67H$MKiFez{oHipV7=xnp>+Xn9s@vQ8 zveJoMSzN>RD_az8Xh-}J5CMs&=eDD>JW%(@miY17?rKC2hufgM%`e;2FF>(TPTM8_ zWR6y+LG)+7Iet{YF2dV`p;R8`)!miJ4A`?>64}peF)yf;G59+$WcAF={MNt_LWd z`{4=BI(Yi3+U)!8`Q!e{&it!5IsfTbK?{v5v)s9r=$d{&y^ia60!-FET}b-z!$1$* z*nSoPGa&ugo~R1!IPl{qd|k3+x>k|Gn_{ssk?BlX=5Wp|+cc+7a?V4;zlL3|)_*s**tnH@nmDVkLF z{b$7RsR9ok8kwBOHX)7d@>jgcN%FOmdO$okCSMEMs++5*GSuD>;B=w=cRi4P12myI$Lc7g~z=`I>Wwr$CrKC!5D^sQrH#0-Itda z(g_&}{s>zu>&Y4@t?3%?#ui+cb5iwWQG+#(_<}YYhVj%zSQDNmFd}_ClsQj+v#Aj* z7fIxU1tXuvNe^UIRL&ALS?z?Xqf8CXURfATtLjL%E;c(nYC7$WLdpPm=u6*qWa{X6 z;(TD)X*pZYu3(!ZTEMK>WPr0^b%pJB7HiFb-sS|oS8yZOAH|ctH(PQ3{f*XN469$S z3@cHhBZ@&)Rtcr+9+g#Xr5bMDz;}Q{4qXec`;kHj9>6(`F;yZr7t%Xu7n{tuIVb!= z`2ULa9(uWDn|!+)Z4k2;N`!~@zS_xX)NibbrVwXWuW+x&a%=FMmUUX$-7RX81$1&d zrP_;_yb6~ps58zl{|{*ID$-Be(&xJq6PpkjWU1-t_oW){Gv!vPu~h0>uO`YUK&%S* z+m}?pG(-m(w^W_D6Y2jM?G4_~DitE5(r!vD-GXL>VNGA=TVrWTuGX@;lh?jj$S-hf_xS$^_TEr3;J{n4 zUJ}_zxikqm`v~d!jI3H*tcIBG&Ik1W@2&o8qm065vlJVR^a)L!Sd zH;D-CZ*t|53|t!Bk36xtY}OMuzT4eiSQe<2hR!$HDghx|#Sw7d^5agHeYPOvk#22r z)LD#sR>hDU8=ZJQMDpFDnjFT(4_ahG-3TV40wn-4slHwe3J1+5?K&GFjcSwJgbKjT zlDg7k(~3$tzkC9AEijeeLrQ)sKa$#C#&K_6y-zhiyS?z;?m4lGbY6Ra?p*BhvyS8b z1ge}4Cwg#Q{uGQ8zC)I=wKlg?q=%cOo;gYDqZ};iwXj>L`(-Oy!*}a7PfU1CkP$B* z-#MFKVC=C~EfRmGlSx}Q3W1!Ty~PhLL9o$OO;^ZEcpRr8sK>K3fS4?DUg zsJ#PIBrS2sI4pgD(j4%-TM971uCc{{B+rJ23_Z4E+D-Y6U9~71Fr@dd=}!~s6Pn~+ z*^wDxq4dJO`haX$XVYGYkkD&Xx>d&O{|e9jCIaz)hvx!|7WT%S z?MXV#VngLtA8(u8c>R}CACWd4apgB=^sK!K6mZ={QUe^nLoS@<|>b&z!yV{ zwGDWNL1H9B*Txx`mvo<+2TyjhzP8+-!gCn7A^b6<2gq#`K%z6pHNYg6k!>oHo8^t7FR|LXEwE>PrOrkn7@f8cUqYbU;A@943E7>Utz zw9YXIc4Z#oiie?*Z~2Me5na<3i;Z;^Y7Bx3R2suk28DW1PO^(N>J?{c4wK4X-k_Ec zLvKgKnUkuY7`UqX)A6~UF^*W>#Tm=lU?q0Lp+DonjouUX?)?%CE8?BRXj+NLMqQLF zgy(v+{0&J@VaNlzCKS5!QW>1$wB)9vm@7rbWy$%W#`*4Kz~zY6#gyWPi_evB=hqTV_Nhfi6Hc!m=zt^FR(3f( z5Au6M;0e~5{>&^U)r^9frkE#}^&NY?-E^Tc>8{$PY7v&H3iy@d`JVV(4o8BK-9O)4Qs^?Y>wbNqGynM6*8d-)+z~VM zYb|eEHqIbgkft267kGk_^BCry$bzZ~u=T#z@V=J$*nDUu>S)5_s2$jYxXzoX?!OTeFmwwJ-f6)1WhqP}eReGSO7%Qw1a$~~LOT1aI-k0a(F z4zFR+ycHhJmNegR5-QktZaH3`9Mt6Ad@iNlF_gw*9?WpGXEL2{n;5|KbLtpflZ^ae zKBKvG+;5(}qDjBkTgcC$z{q*gf1TTZdwKW+wOYPHd7}uPF9nj651b(-niZ{3yMN{0 zaNO#~K3R_t%g@d90rEgofUchHi%6HuZ&|2fibxF38s9`EP#S#25e>u$aP8@1F{T;x zM)xzTXe3{_f*4;Tqo1vc{fo{P)bMnggzBP(DHzDd*i-XeN3`)P*}a2 zV;hsJs6bY}&}yA7*OJU7jirqbukz_&sg*&ZOA|0f+%SyRnJm*T>WEx)y|uho;5F}Z z;Om6Cy+kUcB6|lpFe>m63#cdh03B6_;}zhg7Hc+9B%MVr8*3~m8h*u3@ zaT#uWcC2*V#R4Y1;V-{!gbZ7z5dteBk@VK|^z;GEIyEF=zz2SGlQ719xtiH($6cH@q=AGYyFoVib(OGac zE9|O`f;JE_X|@w)0!$C(`6ufG;%Rk=5#rR81QCQNt;742F(~9Gz+6YHuQ+>t?853l zc|u(DF)#A$FPdT&j*#El+VGzsyhI~Ei5MUW_$U}(tiGYN5f!*&uvGBLtSn^ZmNyMi z@8iPTl@tkbln_FJ7H7(pJD(e#z*fh%3%q>IofP2_{IW4lbcN|G!+#lgOSE~wi#p@% zG)n0r&^o|pBH6Y1E4%*z%u){i0%o}Yn@d(jVzT@?zb5(~9?0X3`I#!kqBknL0beVq zI{+6M!_>XWcPn|mL_RcqEOSd=Uo$bBR1%?C#u(Kg3j2ip;DQPD@-8tb$0@q^%uP80 z>@*ycEaqiIoC^S-f)rIrhg!>p!sO&8W}}@#24&Mqju?urJ^frIa@o;M__t(UK9I8= zo128<{R76n7z7WEULz_bi`x@!~V z0|3eL4JLD>qjb0X<|S+*-Qq;xuXp>vVGW%Q;Aj#y_!(PIUJKlY&RDPU0{BH|Idj^EUV0&B zm` zpF@`bEvVbSG&xif;;1M(rg1N`V9KJD|6xV9=ULeKw^;zVE z`FQMyeV#4-&>Ow%6Dv)e+%@9*Yhyowsav^s1Rnv#w}M+MFzqKcR{7B`6sWQjPDsAkO>}K=f>S$&8vklJ^ z%+>^Wq@C=JXK0tN9i5!SuTw?0E@N&rk?=l5mft3xG`XuoU~*+GqXkEH9x*$wlgXzE zhZ725fwf_0nweysD1KRl@hrbW@A6bJn9Ne6)`2s3k=Wn5qxI$~lAgTc zefOx_GB2QOpxvLXi7(NtsoR6$QZX7%F00M3JV=?ro23IcXi-#k_W=S429{j;n8@%} zokyRW`HDkhKmNEbz|l|3oD+Z$0M;O~$^?Q}GN)vra!Fym-T=UQh?c_a6RT6D?)vS` zhUQ`7uYAKm@Mcp5H0h`~X3uL+8@1Orne-joZAm`s>ZjD%8XQG37)16> zpv$^AEx){&V~TKW87{<8Db*6gq6Zi9S?6sg1rsFqZO<9tq1xBn@b{9&K= z80I$f5h!Qxti@5P_Zd3%1o|m}L(w3WKYumFpT(q98_8w+X#1BJ0C(Q?=}$?ME>wA` zvk=wvmU}Ysp%g6KVn5F6H)QBtF1T2UxzY)8?~0@Xe;ZufD{j?9 zP;X&Roqz0pS6Gex15{mhe7%g{uL&>|sh7fEX3%UVRd85N&JXOYuN$rST`t8a_r^#~ zT{fu2tu4CbmCV<90?QZ+hGcgBGP$p28&cJ!erH>YvWE}b#x>auYKd(2xpLIE=c*la zWeM{xC-A~S!us&LLK{5O9f1+q@9Qh0sXVxoj^?ag@kPVzwY z0I<;LPx%|U_E8lhQWS{Un6XLW6E=gM%FRlP_>NMPo)l2EY?5i`l;5oHGJY!5C3rqfQd^qXQ4}S zq!W6=QyU$<;JK(Z>b!NE%7Nu_;U`!1Ge89vUr_Vgm3EGMqM}RP@Us5_@e9Vj_HH5v zbVMzr0CRbqScIGwb`FJ*oAOBRTSHiKZ4W>@f?4ecoU260;w`Rs8G>i2T{WF_JX_7yeZ$o zEOAsiA$Ve`u{_xcZGXo*pg>+70OT=C?sGxM*T#YjJF!^>(ZM8bz)z{)+64TK>4Pz! zN;02#BmU4Fk%Bhhwd!mZS5gS(y>vG$m~daQ=Tq!XT36bAC(=Pg?_NDWT45NyNR2<) z{Dhh7@@0%#rKoq>w$-$PbmWcR^oUxRb=O{fgAP0{4ztmC8TsJnZBRNeJoEu5800<< zm@4cQ$0Vdu$D1&Nj(~G4sMt~ViL`d^2Y@zR&s*aWtzH(wo1oYQ1JGY=v{RkGJ5;#|%_OS^Nqt_KZ`F#`(= zAhbXoyF{eL?DVH>G`S(O%i+QhU{+yPU)W`i9PEsp-`OfOQ%llP!OmedsthEDThl#q zy2i2*zFn(DX~X#&%C=#FJoC1#LTX?6wUigthLo^N<_e)>Kxp^i{%T$%&#qo+a56QC z&+P!G;<4zuYv}`eZyk(IQHd3TNmJSpQNDVV(V=)}scC9KJ7qlrAH#?fJ&_Ag6Av2! z59_#jr+_qun@LSy^!ajDYWdLSoYPZ`l#q%n@dy)noFKNIj_B87h|LHYvgD}JqG3y+ zK1#o}PW-8L-9_z8zceADl90!16p`n+9KgNc^@stF#>il8-Mhw0%tt;6iJBe9}5VwJx)};yt+HfN;@MzQ(yKMC$Xs1Z&SE>`Qv?42nV913Z9H{n?q|;y$&PG zT3^vS9ZpNp=%;dl>j8=(3|TQtjr9`0dXq_p`d9SD8-eODOx%G7BQrKnb|oS&0-;`W zmJci~x0eUf5eEzFxNHQ|z)GrkO@EHNaYwYr-u`a7!kF(9fj}X_jHXGKHTvayFXOIU zc@}V$g_oz%qKxJJRdVic#eKnS&>ysfV=E>_lzzXzDs9sUiGllh4-e5KPW&^~BwHsv zdIE}_2f=rg07)+v$VlzGJ`Kl=O^FDN9y|I_!B8E_&d<*u&8GQ*E3@gn!Y3|XI_b&c zXkHh@Yq#zT>~Qn&7!GTGV2-RKJO2CyXt*#FQVb;6ubK(SH|6Mg) z#srGGr=J*}q5hg|f?Ofrig_d4CpBIYNRZDrS_`DckZU(MxRxL7^+wW{P*>cyHNy@6 z_*>#62#ukOnGFv+aPVj4VOf?qlEj!kHLxp%!M{=n3QobDNs#lG8VIO5IcEVfDVHoD zh&}Dk@vc3h4e(A~FYJ4?J6z;@@&iGBw9DUfj(6a^3vW;Rt+D4o-T3EulNBD@pGDg` zd>=rPAbCNXG;gJa2STVrfY8WkK{z*Aku9&ThOJ%147|}Rs!0sZ{9O}e;)aC z1=^BjiQer^7h1TzIX*cB_k;-c%7Yf2>4t`Ip}rV9Q{_J7Q(%2ooM-TMJ1y@S>Hid zKDK&o^o~(jamb+n3HLhsF;X0@UfVBz92WQW52kdZm*=+IB0POxNeAuUW-Ls! zV{AuY_exms*TMeYAP}T5Q9DQ?k|-*O0qv)mXtx-xP+`zyfzuydl`g}!>+WLPM@!e%C-(tdCqnwijKCG zUC|()&a96OMI>7?k4H`}|3Zaj$VAIpSD3WRDW&GtcbJI`dUBS`9nx-?-8Z{g^7;YDZ1&=BqY=*z<|00O zjrn@s8&ZZ5^8A$MK4;{c3Qb~uI}G-qtPQ029)#m3m&z;I5%@d@khaY1*J@oOj%TWv z_kE^Iwlnq=0@NK}`1u;gfH&gaBw*Qz0WdhjMsw_Q=4!2fbUnx2+Wh9XHyQKPefJ0G z6KqNuKz#9)s?Fi9l;(THp^SXmophsA6U1M?^bb?W>-CQOw!mQM-)54=(X5GaSkIShTpr@E6lAMX@jn0>sJ=)hl2#zw{*4NCe65B&${)|A;4^|K zkj}Csv)2J_fbV{g1%pn@7noC{v3obC4#&iU8T&}pf=Y+8Ej`z>BfYu{x>N+2)f$A$x2Ki1=a#OVM!ABxVPTjZNw zY9zcs@limjlMjeL)(_%?=b7t90g)FK_|ADeOVzr_F@(kkcC~*^Jc6n<56*BnI_VP@ z^_O3OGJ>>=&oFd1KrxD}=K{Pm%*j{wOSzlKI#%(xSG2$HvI=c6D9sB zChMMOJTX8?{F!0HEGXa{qg)jJU!8?q*M zb+xIpHJTIh)o_gbCf88fPBz_#-Fj9chC=R($&bu}qXMhgRzV2B1D1O*jVtEju}WTb z!!WJMrNHVQHWrw}HP}4xmT1)b)>zG00Co*EcyVP4QC(s8ep0b#nE)921JLA{bUu1+ zoE-I-jFmz?nTXX(A7KM*mpZ~!VblbKJeoxvjg@$@SRF7-fwRnbu*dn}2uoNIpC z&Hijfs?kuQ)ncs3-0|x7JOD{3vS*MOo~WL!w}-pubCmkhP*Vzp*sk9>!NwPO`PE_} z9;jXmJ{aF|?ZK}4Ii)rzXBpAC7M#o#t5*Yol;-&He^3?xXo>ax6=PNrAmvx`dlN<} z))(8we~PIkK2N#7P?h|Fy0jed@l`A%STOyzO4iq3l<$J6Ki6ob4Sv)=qik@yL@Cp0 z{UGa6gRQuhD_dYhXT5f6t{jaCT*5u{eDIj&J$_Fwm-xf|CDZ;=t*VDt*`}+cGJo6$ ze4eoj%Va8@2j4#Z!HYBVLR<4jzK3kq z%Qd1bQT<@!zqg(SExu_?ZM^t0LE|y86xbX~F00XA7fmib)w+wT#{LO=2r%r;2 zTC)Wm@b8kmm!%x^GIqbYT-DFD#$hphw16u!imd$0c!@Y6H}Lq=sk5dhx1Sfj%ZhiEK#wR3d`ueqYV%+* zWS={8MX#CNgk*lS#G|FU-Uy*-X{BEI_W5};0xD6^YOU3R^3<06gO&C8x_SE{yxJx` z-}utpEvxJJqcd(g6k~TC+UM+B_Iu4wt>3j9&NG2Zu5A)uqtgfw24COO1dJ$8Z@Q;3 z64i$hTX*>$T}$LdJF&o~1Hm;8dteN;iWH_+R&&F{^OekC>q8f%?@=fKPbdL)={-x2 z$cJU>bIET)@xF6(1bH!la)>4rZ{umfqS`!O;)BZfK=PQax-qtK;nWw1&6{#ysD-`S z9?KhVHn#qRMzYLIWYAZnY#L51AI%!fl=ivWOlj^NT^M7WCzQKd_6Q8o_?K2w>hv)A zN6)4?O$6aMYSo{N#jK`dJDJLoA4>rTrzDM9OY=RERsd(suFR2obz2UZ(LUah33xN_ zg}tTp0*e$2Sw-7oP`o|cdi01Lv0gJ8ERgzWcSpjekj2gpxNZpctCr|7yk_i*X{+4m zuX?7OX`)wZddU4nA-QHB;ZBMQH6oU<^EtevN});?NgygsS5D~LSUT14vN1rn=}g$} zo0)hY$$?lDcS%h=TbjxhSp_8bT0Od+1figyf*W-X1F`;u!N` z<3ud^nw`=6E42tX?`jB{IY!hU`c;i&g4QZ)rsf=pa}WpOv~6Mt63+5KOCy< zYA4{vrF^0lx?6*eI`DkWY_#|z`a{`|CN7rxg=r~w25{z!R4xxG1oDS7nfnFda7?AD zDS@FplL~aHm&1aaapf_jxc0TGM7 zFB9P**UWDOQi$+pW@i{|*ymCFI4JY;z6~jmy!1!q)K}FzEC+bi`brGd)W@oH88Tmtlq2{Zs(v_Cx#;X+Ie&|`9OP#B& zKb$9Usl(^J0>~&~4dDf9O}7nVxHtBrob-OR?l6cyg4epo{gUr@AVi6cP|C8#JSoDM zD_{S5lz=IYs3)WkA&!{$=?@k{b*ZMJyb_c_y~&f(j@ABtYzFmp!Thcibry96wY|pq zt->;N1DmS5``T1Wk=b$ODh(B3dek6B0G-Iym?%>8YjCV3*=>`er3Oj8<<*M9Z2 zA-h|qPMnJQ%wbyW$=gZ{|IArxGS?8PZzC;kvP8W@xU)93b(366ODnckk_L;tWq%|M z;JiqAxSp@)c@KW}8q{aqr%~A%_qQY9nl}Z)0NwFcktS_Rfx^t{skf>mx2oz`p})sm ztrc0>s+c--o##FGa5$^SBwp5hWJX6Krf}1Uc*sxLYg66HVs(|Pkc78GQE}kr+sZ3n zMkYr3V*fOA)iQMG>@L~Qb{H3>ex?F)fX&sgWN)U}6`Z5+cg)rPLlUp7J@o~X$YDWx zu)GVpX)>D=|6mHs)DBX26h?+(T;aR^J5?_N4kVuq!gGJE5SIpqbRoZMS3H3aGKYwj z?qAZNvkvy>AOwCoLIA=`_XN6$MqgTnqP(f26rZUG_W4L>H#p}mH4HdMb@c#x2)e=- zbO&PJrJU6*@w$P*B}J|C^|N8jAclvBJky_F3wGQbXk_so;oMY&vl*+`q};WeYzP2G zmfXTRoU`!HK^^aq6qWXe1MNA}`a|?F>Mrgk(u0CKGLWe57DguP#aaMCJXtai#rm6i zAVlEAsn@@vbqwIt#S#DN45HEFXXF}f(N2Q7UH1&N^20WQHPVb&W) zDEN1b>-vdY9@Jq3*8*5?5(-*Y^q?XrUS4T6IQUGV5?pmlJZR=>!L-QfzlqR2Byqkg zuZ?wW^Im!}OO!pB%;G23WU54QdH{$WwmT|cK%#vT+&pYlkY-~NG{G@qymh>Kj6G6& z{q-}lEe|O5_!+bHqW7WP={|^q786)GfglOt?>1{dZzNW!%P_@dLl`IzXIUDvCin|e zF5FG8W$j5ip?m|3)0%kKWG?WyVkPXf_=Kc;1dbSQk@tY8TxG^|3c$^)CNSB8^ue!L zwX`=}W(gVF={jt1m<)1N#)FJa+AX`|K6gWD#%Pe1k40%0aWwlm=y!a~ z?NUT90ls3UIKeqw2dkFpw>y`w@9P-~6>r%~wx_8CF2JU=O4a1BKIV6u@e(tMTPAXpV_U4 zrZPR|mhXdb_}`OKj5Y&Sk#@F(rX?Xnw}83V#H?dVh|xq}+`7|_awKyl$sWAcJK{jV zw@pt^EQIDK0X<0Pj42i~L=Bn4O@QdyxSl6HHo<02$ugI~b&xK=fEfO*H>~056}}mwrs; zEanya`mG<{5^y|FIz;>`+JpN4G53~XRj+H?uOKNk0HwP@Qc92p0qO2e>F$yekVcRO z>F$#5MjAo7k&^D(H*?PQthMG^&%2NJIQEDA5f}^x|G5A6bzSHAJ6o2D@*#Pd62%>7 zmtr9-n_LE{v@0TWq1YqJ!DA>J(urX%?V-Pp)_UnEigUT0GxCpACpFNc<$5P=eZuCdDtCYixWsNGVt7Xm-68S>bpyY(=Np5c`&C?Feie z?9M!_+dHmK*V=qh+dFK&7E2s9=JC`FJ?JKIMu`XZJN=r8c6qp)R!Y`s+H@+h;gXL-oM5^>o)M^-UP3gX7XZ!=e z2%N^5qwF08W*Ksx>@QKfuyNAJ5@r{9J;J1=NIwy;U!t3tb55Zds!5$aotbnllEYW@ zM`8~kTlWO(LM>@in+3cO+;NKD$27~|P}Uolf`kN`+1~Udr$|EY@@(Ezkp!A`TpT?VKgV=xKu%;D}PVil2>uxLs1P`YHY6|rwhVqGJj>H zv=}nrZ`gUwe!6_mYp+_bA44bq0XCEzzW!&vd{)81XBWW+fz~mp4q^E=1+Kl`B`gHE z7cW&6>nking=uWeBTjvU5G4=>g%+{q_ehbSEv8xTF5F%m0ry{xW{iaq)^c)c?*qgNC zZL+fE>TT9vr`r=s&E_wXHx7!H3X>;0*#jN|e{4A?9&eUtG;=d(F)x`cwewz1FHKAx zq_OR!6c#$1AXz)nGCl5F5HO+j8QawG&Qgt4!?YM7`UL2@^fj1xH;#SZ1D3cOyBLQ7P$5{lV;uf5ixkmj@(*{T_x2KBY zRT_Ov?J@OAAl@VzuPB~yaPs&DKKU%js~q^mko!A}w^57h5m}UGjhmO|oVY;PEi~|7 ztiHYid9L}{A?&Y_3ly#@_wF!fr`vh#vPzBudsryUQh05kSa(3sX?e^t8I)tZHNR~> zp3w5CbSU{+63+&cQ)jg5hSxWS3E4N$^vHwn^}Ce<1YI>8&txWlOlFA&dZZ8nm1YpC6=m%p#2qTF2`Z`|TTTd2i$HQu^Q~7E)yQ^=I zoXBy>uG%5coaacyCAFHKCRB_hWk^%3Tg}~TQs!fu+O?nF7Eevw5NI~wq-FAB@6e?Dq2i55}iwnWV21OJI0O=huwnUUUvl+ zDt{2ISvf7GgKp@89|{i5sJ3X0nOUrbkT;Hfpd$(f*AEq0NUd;E@MPXhmf%*p%M{v8 zpJehqDOG$>UIW*#$lw?Hb?0?ygn)oH&ZTN(vNjSn8lGaWxlylRhl}6Zh05e@S=>B5 zOkRI9qxdNQDF5~ie%EK3vI(Wd=j9RO8X$K>3=JlA2x?|neIu1lqNjD+94ZwGwnV_) z33`rEvl>W^ywB~psznc!^Bd|PYpoVAW`(O}bS5@+t1j(<7|MLHn#H5@>+(2&r&>Rm zd#BI9@!rrBHSWpyyh0&yHuKP7tj6787h7*%;6#%ddEA5^HnD#^T!3HH3>fAhFx_5u z)64RDoXPXsDo#8}7o2R?#J-2Z|1zz=5@XzhDJ!|w8!mCjUBk6v|KQ87L1(b%pEr)C z{nY+KkJ|OBR4r$IJ!=ABc>LKFm5k3$O-7^Auc*q>T%={5nK>Gn8viQmeIExd7)O7JpUgL`3;-@^acx9uhM zoMBxUp5&*NoBr+9*J%B;Axnq=a=sqSHO?QiOGf;nr!)|AJ=vEg`jxS{@MZZlam@HC z_vCSDZn(2K^h#Q()lLbOVXz%A%4B1yiTlYLf;hv z16mtjXyRBy^fz9rsOVIPCDzqn`n3+0(@mL=Iyl!7f(n|#wUDZ7vqh4U3JaC@fnqfJJUqLLja+G5!k{pOv@A#$8}CeeEn|;e<^3a zmQLpi<(V2v0&12#-5{?%**eca3?f!JS$0Emp)|nVDKXJ0mPUXV^Vs~j#%el}u}{aZ zALMLRpqC+Zy7fHH)S?$vB2S2()oAEz3%|}GeU6v=#VhaLN%<IY*h24I>Gs44#v(Cu0Bm5K{wC7Qxz!wS@&bGS z54-dG3-xR@mb6)=y3m4Jmd**#M!gE+&8_^Xt;j!KbLq#A<+M9Bt?XBma=N7yH{(n+ zqUD>{-)EKOkfK*;B333`&dvTZg_Bro;zYx1<@F;(V1_Qp(0DM8Q=@k>vuN7pt6GI2 zy8WnB67%9~lZF_cx6jgc2)y142Tv#1VP1x~WL=-d?%w;zE;`}TkD2BS_-Y3oaan&g zD2isyFCNJAClE3~_kSXSgTOc|;yFe^c5XjA`Oec2B@#huI_$tGm_U5s8LM?+oSMaV z$Z_z)+If@lDY8kPY${vQ_?0PSJZr!38SGnV)c}Dhg+&(SxOhbSm^&`J;_KFjs8Ibc zR0&t*r``nqu`_r7p5uqhw5uQ7nBX2xQWw7$<*x>EVGAn( z(+5jz=Z8%ft3W&c=GP=M#E*svUUX$iR|py6ke=bxGc+aQEqJU$YXHv0C7vt6xUrtA z3~4qy1PW~E1FBRDG1f`R1(KNr-IrccUxWm<$oH;=U7o@ncfRg+AoG$U79fg^&hl2R zN7hz~X!hEH8aBI6<+kdPIv*0^dmW(MNn(FnbwJFWC$z6`m%3Ux>JRY@u-zr_g0L_< zaa(>!^1@s)1x7lhR@@1>!7rB&(aPU1T>`tYV{zYjLkCF-(pMNAD)^hus!9+9lBN*N zA>>`t^{YA_ufK~^;`ws?wMYg~#@1gY#;LKbZ)H4Jb!%hL_!ecFDplqkIg*GC>8bX( zuoli2X`V5V?^P!VypKQ0}I<)Vbg#qSkrI1O9e zW5qV&0-%;ZzG`e}7{^$8L61P{&pXUd@r6zO)%mJmTI$qJ2;worWT|#gauT~CH}!@S zka%a?X!kBKeAcOH=$eQL1V9LTd{#(CrFUW2nG-$-i*T`U0?P~2@-o#(;@8uo1MIs9 zh`*0%GTPHE?@V8ijNF>Ge7DLicX& z&9eIO+J{Bo7IHr&xi=(u%f{x1%WZ@6GKCK#uH6xYVqQ$GNULLJv_~&z+bm^rKOQ^_ zAUE^4I+lq#@#T!zzWQ3+d@pa(-t_afqoKCo^8;qqLe&=XEBtiEy!eDT#9ME|Ys{tW z6}Y$jx1+}81Z?Gz6(O&Fd92)W@lc+%(xD}7UsN%Ev)JO8t#o@zJ?wS3+_Lyx5t^ znvmy@L)?e+#$hc_AU$BSJN9YiMaSS&;Jx%Mw_k4aW~?$_>I?64$>X3)yLywiQ$a#@3HZ9HQeEB+E7tIVLZ0AqL-QUk*Y38cK2FfA zY+6+RuF;^sW-<&l(Wbn<;;%1V!Xv z419VM1vLZ_xW{dBTW7Zd2sRUNM2t8zFf%t7Bi$wDxeiGvbvfI4QJ5zrK_zwn`E8t3 zBI`*85c!~+h>j~>{-tD_ciu(xo+%Ad$@k5&JPqT7iXB zal*~Ok+3->C#F$3U#&j@3qqe;z^`_~e+$+9-6#H_g>q=%sT~~CZ9>4-qmwp_CE?uI zqPafF4$(lE5diVNnSmRwOs6F?-P7~^S>cD*USg&GM6G7J?&+KPk!G7oAuWs7-)bml z?W*Z#nP24CuOs8l2C9oN&tdVoK=f{v&%M;F*|m$#KZfAG%zfs9pn<)y{L?QPBK5}O zY=?bMp`Lu;5W~h6mV=YmtMJPEC#o(WP$DkrP|e-oJN+z?n?f4-{LzpsPah(;5Y zLI}WuqA0eSYG9vIT$wGJw4rcehw1=+8-G{-R8_^4&X;hkO!wJM%`45BCkPER@}p@h zbrI}GZ>I_Qt?89ZRKi~wKXVP9%5$+Hw?s2OoWm(w#`8vMC&#OG+{VDtYq_RtlLUQ~ z7pK}=!}MQCEO&~No*Q-_Z_p3V?3ixHP)_DpaB7slXQ8xxr^iDOTGFK~MxTAhJzksh zsxLfnedAs~>SMctPo*r-0F+Zhx+&!qcD=_lCR;xr+uGGE!(ygvN@JeE<>}ZCtcr4K za%`v+->Z@07E}65)+TFWoxMGC*e=Q3d?pUQHYkr}3ycpItEVMegj?jWX z;f^-OuSfJF|NZO4uNOl{|15Fhu1^6$1O{3|$*aiPc|n=zi}1~bXn=k{4jCyF#S()? zF~wa;#~|b*B@kQqKWWODci7AgrBWRCv22`0GCD6-XIu5@G<#?l-l_X(JX`DZlHKY@ zm^)3GN&pVSnYN6_Bz*#~1ZQZ_T?{a_-VdO4A4YARr@l$4f;GHSk*02zabxYwnj5DV z_cooh(cTDL-Hv`aIx3sW*Qw)7RDbV5a|+7X*VXW_jh;(C5%`}ejSYxKlfB($+Msk; zKSVE7oz@K8iOWUAq#gqRq5}C&IMI@cdsDYU%g0p!`fF8EAZ8j5_w7RjcdNnMXD4?1 zP=?${Kt9>1482dW<8sg1kk2D~bVwH|0pkq-PjvE4`3-1&J-4T9lbdr9r-P|J>-f11 zJ{etQYCG*3&*G>DA5Ll6Pn6_F8YA~a3y~g0DF^}e@W{i6I?NC`sk~Iw;rfq3mwrbA z#z8N$82qP$nqS?*NK^8FlSUkDhJxl`R7+^FDixX5>^30cvBvg7&(<1YtI`;pAMB%+$6@39pW`|7 zBnc6~LPXVcHSv|0tAdfE)^e^iCMb`Yp=b9n6D5R{jp%BJx#BGbr8Tl*|3h@qDBA%+ zlXdj`n`-*df$kuRE`RUSmHDvuK6gFg^j$`MQSuu*&-x7g?;Vfz*YkZZkHrHcu=?WT zgZyrp)|4Xf#J;Pv_NltAxqk>s%i33enPj* zATC6r1%p#Qx+2{ELcc@OGlXLLSkE;phvJ2^Meok>!-@Ha zRUA->i*&N!g7kPj52Nl?R*piHeq$qugORp;#ufIr)X&tiwx|i{t>M|QbWG$zIm|Lc zNo==1MYFz|M1;yUfP91#5=OnP|vE~};%SEZL2U4A~7`MFDpG#?;33BZUjC9() zV;evw6B|FlDi$iq1xB($zHL4ajlvl$9?~#k0fLJ-6o*q)6hAFABgCYjca`ipS^hD% zebm#ugMl;<&gPy>Z4(H4C?}wsoV@#L@>(cUTkL?CUMB0DqyD%T+x~w1(_7D$@pePbE_xyyy+zjW{2yIFNp1!EehQUrD z?kB9nY%H3a0g2#rBKU!}#g`Sn@e=eboV-LSOhQMYbR)?Jjm{eeuQ>zw&G-_|XD3+E zJT^QhGGXud%qi|T3<3tACgMo4+dwfX9Y?3U>eM9o3n(;(>a5O+c5R;$Uww=^PGB)C zC_j#$AMXshP+lyej%8DS7FP!%j-?J`Qxv?6m=v@TJl$Xx^AAXa6M*VKX({Nz8{^4J z%+sFnM9Pxrk1ol>A7I8{sXCAhqd5OOMCC#P52JAe%qma%t-*KrXrba^LTH3@hcc3$ zZW_uUOo$pbY*|bf2bGh+6X$*qbya+i&tuF-WmjKvL~_MsKc{A~q!7UqK&!2a*c@#N zSSV#nQMIA&V4|PtV*Y6lj26A6SQS*k-N*{vcZEk?irK9yMAbo2rq!Z{P-^nF)AHmq zBD21;=y*Apdpw@6bD);H+GDCn`3ec1gjwx&$(=MCFIX?l_>4)R&>7VZ)()U=A@LOn znJhLRNZw0n)%_O1X#VY->QMqj^4P8W*S?Ed?|PSS)B`~*`cJBRqc|9m-=k_37P+hcp9>yOFPAI&R&R_rrm}1YYp}?}LBg3T6H+lXg#oVon*JddqP5!S(`SG$zf$7lxFxC-@r7cyV zpT3x$Eo$CSwMQ2MF_fV3{9)Mr>1{~s^{3y9^}=O3&Ck+hahn2p0?I9l+T+U&AC!x7je8}Gin)b-VSDM3B;RF-yKZrpZ;EW)9A!+>>J6oCCg>Kt@$5UiC)NG z62?^TV%ziT=x^m~rI#CMfC2XRa!I2K5#YFFqq^rBPEpn@4bYjNt0(K7Jc0 zspjf-x5NIFMxmMpK#(Q(HR>`!*{l&bUMQ`(Gw1V!*axt7xZb=`1k}HLk&qar(lG3W zCc9391uza7@1Pv)W0fW1i}3zTDJzO2d4|Vcpk>A+(QEgq)v0mkCJ`uplLL<>R+oB8O}e1h7P~QXKp4B0y?Fe+)h6JUTQRr^|?MpEaUe$ryu=4 zYYqE&Jy08*_vuOwwMCOTtkjX85zYoYOD7{kpN7?vnwg z_~T!slYb?DVd!V~f|~{1ap|XpZ))a=7Zi@@2*`gch{_WlZR z_+y}xVg@RdYNtVyEUCoGjeO6*Pi)qE#7ddj*q}5FA>wltAiuwO86Si#lI%=KBj^WH zZF<3o$GM3H&FUabqZ<09ghnRib+zR-hIeATWExNXvq&>xF8j@FkcmTw_I01lp~vwg zmtZymn~AT+azP{2L(sjnbP8BQejOAjLQmB=5z0XpF-uFgxtj8fL1y7sJ*Z>;HCcW% zqAnuhMW-Ew@T8hwIRiFTCQ}j`*`-cuc%{@L_0+Oyqro6Fuk`w2zeFTTG?vU*!Z7{n zSt&V8NVA9YgiQB~^YeYpovExh3~EgxZf-Bw%M2`4OLS5LlZTg_*_w=q@<-a9Q_7t0 zEd)u#(MkJAUwt05FuA)v&HQ0C7ah;-d^gLU#O~104WZRK(U4fCEp4(Tfn!i_it5Y^ z8|E)&vz!lTYLbXnY;tv6KO}uI&Y@y=8E>0FIEaCuOaO)o5hG|0%0GSZd9qkEWg5gMq17R@G7gtCaIM(k}+% zP^)Eaw7%Jgrvds3i(yYD4$J*SyYv41I|O41Agg1P_*G_A^hGp5q7{mdUI7P|uUJa) z$`a@t*GhmcPBCX7ZhK&3FqXo&4e+7 zBOif-rE+iH8tWr=o7MRrHnT#H{aTM1K>Pyk8AAz));4I(DlwN)v)ha(=(KtZL!Myd zC;#SP(3E4mD6Y2tj@L4FS9@|X9}pTEddy+g&AGqi{9cx;$~cFw*}YPHqWEf>M|7eE zV(#}}^<$$k!&6h$bfHXt>BkDvT&(=Q`u>d09Z52b#ee09QzaCPef@r{iPpp2=zCI} zgu5H=27&taR3lR;69VSbwGN42_0DH>NaEqiY#UU2+VB(JDXZKC7IJA4SPUKdoDf!$ zmRr7K&VjV4aJVbv_0_Xj>gZZ^hpmYrE{Bp9TY@SPD)`lXM~7L9xk>L&TA=r*9O83L^97 z$C37^bU696d9Eu5*!EJvPEnF;tW&L%aW$<~!gyZ*I8{wU+VfNw|OoQCi&o9W|~J{mkVD(2${YiuTx zB2teH58`sep+(@UeVK_7FuOm!P}PCQuBTtLUTk{zJ&A4AYAVF|01TyJl&Mk|v7Ap&jZu;NEKrI!DUuUzBJC-G?L2uZA z`OftLhjVepY3rPD!%QJ9jdrYSo+SH@i6aWaB_D_Re5^`|njmas0u0SPqb`CVd5%0&gWb)0v(9rN~^1K)cY3qdAxgdX9fTgTw`QZLlpCBNUp zQ|6&PqumE7+mS^4a6OxN<-V2SeN8pBvDHuYK}cQLUdYDXe}A-GdV+@1leJjN_s8>K zK0m?wLc$2oz=*L$t-z#bYSm#SX@@hBD=2a$0L?$HjYG?6K6^ zkt0K`9IdtKYXSfC@++1&)1ZQVl?;yDKXV2ip6c+=d_06M`!i<%*S)g7(ShCd6HG4_ zL&^x}4Mc+5Yc3I%RtmGl=Ye08wfzxFHSR14pj49a(4k}rJ-Eg`ff+16r;0BV9j!7bhX%qB4@-Ct-1upqD?iLF?+@5S-u zm?2G&q(htfrs45)`4JApxMgzf?g?@mmunaoZ=%)*PZ*tYB~9OTY*Xz9D`bA ztn=RDAeZW>$$Q{S~;FX(NObrKQ zoYiT-Najgcib&s$pT#1`xF%7Kvgw^9ky~;2gd4B-E{0-bW2#8i)$ID?E#u`wfWsSy zS%7;v$~6>=tfT?J>rl~qzNFjR5k+foO3gW@h24ia@b9|dNsHefUB;}O=kfb22iT{rzMMuMmD2-Zt@)I@>)vVL6 zF4@b$ScbtWWZ>-RHeVR!8LhsWHs4GzO3|uwo9Nj?@js7e)Ge^Kw4x3~`Tg2{y$?5A zV&FNg8?lLN$MX)#DP+z=&-aVf==+N`p^@!|P(p&@$V9lVqymW$8Flkx6@>6hvfLA?fU$8zvLZS+Vik6`)3ZH>z_o-kbC5ODkbnH8e}Kn2 z|CYa8ky_UN5))Embmz~9>#RE`?M9CB`Eo{lmhEeZtWn(&JI~DbOeR2(LxPL zQ^w-MKg|u%_dc9kvz!IALr8~y8KPVEy(0K~S-xk-^N7xzWa4cZpW`YtdSkk;Bc=~#}POY+cf6Nj!3Ro zd?2N#DW*^M5U@_!pR2`Pl>}riH}k#G-&Nuj1_O;$C1dotv(CL>)mF}Mf6^v9B(GE#}A9LU+!Tk9{npdCM z8C{ob?nUg;GpK(()V|B;2o7jC0r*^J2oMyhpxn1O!F1x27K4DGr}MrHK)-qz=JEkR zHdd9gyc^Sd-KYJtyQ_1GT-9^ZYUqB~lZ}y)70kI>%N6Eqp+KY)=_Ba41L#eco#!LD zr*Rm1Dka*oNLwRYqmiQoCj&292;4zf^wSO8m`^wU*q=~yo_%l0d>bl?&ta1T65MQG zsW5y)zM@^G@&f4~p7*7trPW<7ut-7x?m?2`BTjEtX|&hv&-128wN%MXJyUUi7K&B! zdlE}3>{GE)Ve#u9PV)<|wv~${T_qTl`orVr5`7WK;%fgGq+9-dkWLegVe!`_{YM6v zq*s0O9m%Q}@T-_CQ1)-GtiF&LEK9a~kBNasz6W20MJUImCR6mYo|%D7s9O&JuvT-n z@pHc}o{R``dtQ>%zpOx0OfdXdo(?7y1}Dlg3AOugAf;FQK@p?gltQ`6yW&q!#-u!M zn>;SQ0KaM5iVEoR=4#ei%YWCX`G%rsQ*mbgg=sQy)IagF9I;yf0hj57o!Aej{e3@} zX4jCvcUljxFc2J|;D{K%bVPl4o+)V?fGw%0_{QDr^!uTQJ8}Lqd-X|0Qc6s!hyKiG z)&ly|W!mIJVK7IqD!S60JWjKYE$Uq;VT7Dnm9ZA#;D>qvVE;__H}+5!n( zElN$aI>KE_=j#jXCO(+u={9?IBO0leY4kI^wkJHjly027VH5L`Vx5*cJ^RsH!Y~$* zWjyw=fF`bUXzpq6M6Pr%H1Z!a-;tDd>w(lce_rcyB32ctN+s-G<~LY8<)ECA25}gj zj54k_3j6r?bK-tms5<4QBWYu8-d<)aepgLa*XL_(P;}eBpS1j*@wpf#?ATKm|AyS@ z_LzOi+Y!ZzsHoA&T!qnv863szdexQ;zbL0%1V0oT*iaN({^M#cLJQ7D@jgXlk*>g2 zVq@jeYpP>1l+d5yz=0RC{!$|#WId0x^KXw7p*B=s8pe>Ar2Z$)<^jv)|1)*XmHKPy zO14L?$>GB7a-Y&o>pq5Ma&G4^C}cWvt=SAEh7%#_2Xm?vf?k{Z17&z-XfP9uT}i@3 zmI#vL5lgHBUIURvkfn4HG8C!A>PkzH4Po0w==Dc7#JDM=PSd5t%s@Ah4nhWNofkYRCI)ey?B`Et`aiP3 zj0ipEaQ0Pf=T?WxJ%vR&%7pV8H3Z;!A**w&NcR%g#AYdwL@ER(8yi-=n1Q=>r!(Zr!a9;HO*t+Bp7tgH9sJp zKq$T|2cZFDzQ%$5d)n1d;RG_v3>m^%I~g>Or>9^Fs45ci$4ulO8{z;As9?qrJ_M#s zX`$dT1LO!^R&=AG!GYR8>!#N-B=ayh$h5D?kdJ#ve9O`(`;Qc3V$eg{8fq-a#%+!0 zRS=YDRFgV)iwV9g{7<3PW-3DB6fflqgQb}V7flsSXRGVhAtGXv09tg99hzK#$Q=df zA>HCtGFXBmu!=z%QTu*q_d-=OLgicj3qpN z(pz|s9)$6au&L)Lyp3BrD!y7_a(_cs$>lvBY3gk?^OcW~$96fih0p#tLu|PUMevna zcC-Hb#4GfFu~+|js7b03N%s2Ks%gFfKnWjaph!SF1ypPJbt;{A0x$X-{e(Q_y~P1&10gtPBtfi2d+TR z_XF;0{)PA1rlgP~1d(u8;TZJ4uep#}7$EvUl;P*rFUxX*kX1WO6JhCAUu3#15c}W_ zts4dJw`9O!apVtCw=hlkaFumB58yYyX(M?Vr z-z^_+!;$hL`65V>XgRcr=%K!1`_~6F7^(lnKq$3^h8#MxTqLc)G@EZy@Jhz;>IFW- z&IR__aljfW0EoA0d#BOc4m(?U2 z5F5rLpTZxydYtaP4Se<@R}7dXD;RPDk+6R41Q>!n0h{w4j{V@CC{Uwj(SVTvXlW?b zU+v7N;~IlzlCDh64tu^tT;X$mZyxqq2Cai1C7N1ISJO*!pt6 z9x{^0v`jcT0pP2pff085_UfcsE43H!A;Q+&Vbf%tl)0uIOC5h53SMt)rJs?iKl2rr zUjDrCA!KsW=4~w$gMB9JN)$}At$TmOp^9*-720KqX z<)g#kIE>(W4Ox>H3M5t<7no0(PnK9ehxuvtifkgY?cpE?tl^k#LV^ zWwTVD44VLZvZdJC;zjSwslQ4=r`%AD!F~?N{z5fzGc1lX9;V;dc|P~+%zh9IeI_^HJ3`bPMiHW|-p6>z)^HqLfAm1-m? z8(deQ8Pqo=fB*jN8$n)gS4(f*gl6!wtn&f;*K3X+`3`JndVX?~mC8i|8$<=3g1|kM zmoLE_L%{4i+Sen7fp9>;jNx}0jV<*kX(4%Bw*Q)nj)Z)`Qd^~w#dbJ?-NGfX!9#ei z-Eu(CHP;zC{qbz~=jSdZPv7@X53myoOhe6hC3hiDyze)#`5bRA?RI7|c*F6y6uu{& ze2D>LC05h#hzcmHNIj!PVrW2q$VxmC_lDDJ`pQE`HkKxuwB-_+K>X@tBWJAjVYruM zAX_}~m3Z<@G@u{gWp@I!;` zgl*sajLt^p29?lFyOmU*rAemk#UeLziZI9<`8Cr2+3jI}&c)Sdtw-9;>>4h5vu}lb z5(vRGE;ESNmvG-$H?%H_^{gWC1cSDTaa(qNitqFr6keXKn6%TAx)-wu+N?|cg3G<4J$r?4bf&8 z?)u&ySkACxI7uNPh~fwe6fLf=9RKxjLy_u5J_&d(c|OnIHrK_*QdV&3m))4&VZSjT z-3c<%N;wNgv%szVng{uS+sVx&D+rB9LHl5q{#F14TWoQlT)EMZhV{;a$w0pl$~lvI zWy0pLu0UE^8hE-HQ&xv%7OPj;y$FU4LL)RPi+j||#JyCN(~3J-zi;-~ls>L?N#*WQ zb0k}F+Wj$pwmV(mkD(0W90oSbC-b`pL1+@D{cKyU^+Jc6okT2sfl#^mIA9LD@h>t* z48+luQx6r!z3eYiqCL0tEoZr%eln82HIk0m?s=6^yN~Wdk(8#>*HS>9R%Wg5XX?W=BYZ{LF^l23l@6#DuC^t;#=-3ag zZ6f1H)SdsNiu`eTlTto8+Suu^hvVs#K0P&YS4w?YR&K32LXCc zq*j$uE$L-M;d=tj%|z-Cu*1sJy_FvaXrBs~Vym6+K|%k7V*g%^rCaXvqc~A57|~Z; zRsbeJ+y`;vpLhA6x4aYd7mUr0-M2v)6_6~9-EQNbowX?$Qt&%S5xr+_Nk%h%Fd# ze4k>k@}M)TDrlA?;{7o^SDjLfX4=Xy|mf~o7&ezg_Y>>n&`l?9XTZ#d221QY$!5$%* zNWo3)S>b4=a8C=58+6lq!0fi!6OQ=_ceZoP(P1}RsM4`z8 zJ_hC$$}<=yq#y*5l$uQR+jEmo$EQQW9#TEI&2B`zo<>VotH&FiYfD4WdhbKHU{Dl z$E5jsvb*EJvLbICC$2j8iv@~HDUHt!FWZWuz8m=s^Exv~3gRj=OftebjxW}!ipae^ zq(N{bDGk+5ZJ%u4jQbS_!L%?J^?6S|g5LWjTWIDm7U#F;4lT9G;J^nTmq}(}N z;7PSuV~#ghs}ePmtzNas^Kz7Dijjv~D@8jDf=iqZC0N)Sf|<Cl{ljbug`L_IgB_?<(v><{3C^n2W#tcn4WFr zGC#UiUl7c*Ibibd;86<>i;`1Pu-sH3ixXSI20Dxc&um`vyW+2SYHMNOxM_J;w~4xN z>IwMZ9nIMUs3)G^I|zgy!IA2sb#$lkdWk03J#)UUCI+@Ltq@kbNv@JC5DW;(txRL_ z!b8&RGp*7Hv5IHZF^YaCYtgK?$WzL1whCiDTa8O8p70@eB;g&}Z~m_!APY9hJ{f5s z84{cg)Af`pM&~u|Uv=0&ZYkT?C&#q`A6+pt7MXGpRisb+mp}3WDhjg~rF7EJ*u=Ug z!&IS4F6eF+oVoBgy*mD^r;?hk46VLoJm^<_wp*?{IXPN}s4oYhhG0yV>|XE`p*<6} z-5$GxKX>&NPy`tKvY7)2CQ|7#2o@dnUx^WDvSPpa{mdNN59fx*!NBw< zeoZC5ETmp(Vl2PzcZ^Y}QzG#-pdKPmqL;Ccj0=6-YfF?`sN8oDn~pckP`afNUR0r!Vv;lrAnVvl+QO)>q%B1wVltg1D&p^9l^qJzOCZ=FX@Z zIf6;CZ%Xc}s>dfMNNkwV&9!y}hV70swXF%P-9e+5&ukcbUr zc1zM2_Jnypt16LFM2>EgZ*<&_S;&OggSbwEx|AH~h%neRvffC-Dka{@i-iWKmm*ev z?PFQzxM~6LJmJ!o=K7l~OfTrexq%(1Zuj2N5UVK4 z&fykLbsnvCZ{Y?*cKo_Z@Hs8ZHB|oOwvjypy$-U0xG1}ST;G3fh;7N9$ip(ZcD6}% z;{-F72h%hiwoT66-CX3C*3}88;WP}#w-l@7yy})`#nx_h-j@_3-C}~_!+j(g;<|6E z{re-xQv({JuIGMbCbs5!fG}l8f!uS1#s3gPio$h$+uw)#%=eI-1i0;qEwZYM4o>a;fR4Pd*!TmRqQdn8=o8xpgI z>sV$et=xaY6^qp>XpU5M#}?kW`~u^rm5GAWfAV+!aR|VfS!Shzl>QZD*eyR{EKJt% zoL(t`+hyClu4o(V4ng=_rRHx*x>3M6HNTfex;~4~VDsKfVA`JS{3FLu1Q?Jr=84jt zlbaxOI~`M?5Oc%=WVBJ;lKsX@xlol7T8moC)75SyZr9}}XWKth0jAKXu9WeKzSVJT zD~)>?&L09RUh`}|3d#R+q!A3ESE$-xR6WPG2o=z&>Vn{3Fe->}8alcCSmK|ANf=68 zc;A2gq8hkf%_j;2r}#G7;>FrJCq8d7X?1iA?y8pI3PfkVH5*gVHJ@&HQKDP-gDzIZ zYE}UlUoS5KOtSyy&qO-a0wGF?*lc^_reB-k_$s?eLR`fHRzE}5%T^PabQ$?9QOa~desyEtwL@|Di{FIo zW-}ka93fg0PL&%-;JQPJBi&DTtZGWPeHC&f6%m=3{63IV!TjenwjxWS#iLh^J=mK< zgmKPy_GZNv^O5}skiEDY>3m)tyCZ45NY%R7wu3Qocqa8TaZMZ397cy;D@_` zjg{!JTy1!XC=#fktT6l&c;Cc&(%7Q6gDTIdWe7BC9T8iH20=6eNG`0-0P%}JF^?7p zKk5wXfd)54RIA=l>~xJ<>yqJS5?B{OyvBta5$r@D2&a(iT3y{}{$KshVru@*-TmXz} z|J3G*B>ZpMJPe~ZU=z#czUR8QR0Je+HwhJj1AP&Pq@(5i(}1e?b{3#dzoSm%I#4$4 z#Z~wLyTH}K^df~>C? zoPC=faTls$qss;3i-GiMh(%Pg(R|mtfMid_larGyx)%c#TOYT*HFp6W_3K8_k8Ubyf6OF^TC%e=Im>)9(Z%cj zJp0QCzB3hTa)1rWjfrfECN^86(sP}fTZ5KS?~ZT|E&0rayvmlAW-n5vTtCjw2}l$P!Bi~SQ7#Z%;MOAI ztoy$c#9?`~?BDWouB@(RHc4w7N#E2F&MFRDsu#$XrO}VGS=_Tnj#ZZ_6)0j3xUJjf z5zf_}*ab7G_16C=(GDGXDkS*_QH{REa;YJ%=@WL2`(h2EADoF;5}!qcymjBG{t%c{ z|H^uMlDoGZ&2FJWx4mH;>tK|ldeB1OLL!A6hV(LHYj&!Lo`l_cJT|%9g;46o?$d5| zsh}I3dNsnxpV@~0N2J#iO_J>`dfy|9;~F zgNY2Z?*dlGGQlbwxl1UPE;I6U+RsisHo@e{G$w{tGwT3XfziXE&+(=y{b}Su81EuJ zOZo-S+v}a<&O~qVW z%TD-aqj&M8G(K!F>iehqd=RQnFJ8)kdS|`TwbAW_irr#7xUQmAW8>55NPA~2ldg9b zat~Ug{l+UM&Z5VR!y2>ch=Ev!vj3;OuZoIm+qMM)1PQ?c!Civ8yE_CZ+=IJAaEKs5 zLU1RzJB3>k+$w0HMS{Bq4{#UB-skMJbK89%uf31=+T*Jlt5(&jxyG7v%+dSk{c|br zQJ@@byri0PMwFJn>uV5U6%I*ogg6M2MqiBTprV&81;H-;Z;u8 z`?z=D1I!BEGoe~e|8DtvapA=d&v_>*BN|xGWtY;Bu*dL+j(p`!oUfEJSf=&KkuYo4 zj?J4KAq3lBO@n{ufA?DrH5rOwBm$rZLt=DqYpd^v@>`tLP=eG*wxn&nxu^L@ru*We zPiHVie2Upm<*V4j@$}s_7W+o@_V0du0td}sxgF0 z5Xl{ZUZ7RZu{-(cqfZHL7LA{IUsrtR!;s2Ug_NdS*)W(V|t8gcs z4O?+?<>B>2MmqG1SI18zeoQ5C8$I+eYYxf9-^8-N$YfotWub=rhrzFGZ0wV0%P{&w z*eO50#nZ$_CbgA7&SGm(PI)9w^oz**&!jUR3m^;hj_WPzdG++KAz=4n14>>(IW=m( zNnpbmeEssG==S5)9cdL->{?ao@lx(swXcO4fStxi?Lj{}nnHI1#3EM7*lDL3<$*Z* z6+09I-_z}aDLNPfgv8HkWaR;B@Cb&27iE{lbA{KGT|Y(43jycDo*s(F?SP_^JVAmF z@pQw7;EUA3$cYYfdc{;mJgqYI*Cr@ZrJxSxPuKMIRZ<}0OXKqB6Z13bSuVFmiaVt{ z(e`MX(U|%O#IHszH(fb)C?Q;!N}P@lvggS}{OZ2(TK#yInEi~8T(ts`FeM&mJ2sp! zO9jOp2OVhVHsyF~Chd>9v_sMUCz2O(NQ1SG@TkpbSVZSXlx+XWF~H@LijtW5;G?HW zV8c|8VMd}5PfH8QnAJl0fI%r0f&OnQSFMf1(bk7q)wCBfLI8p!O`+qz#f2Ce z!}jv4@rmnexa49oBw`AcKg6Azg%(YgOjKXIGUNwZjdKB4z9Wv8w&KrFB1X~#ULQ(o zm<%6J*6EO+Efaj>3UBe=`3w^`^C87&)wAM5ED6k<%W!GF>mql|q7qIF+$4%^eTejF z$h(J8wME#Y|C{N^6D@!TOvJrv{g}FR!zg|<N_i*TKbNUq;FW>He{X&^&kwtXWT{7{9(5<-A=zD zbLsjGuBG&|!!1G4=jhSQHO@nVdsbHms+?||{nZadc;vqnY%JX|`=jdu=6R+371FpF z)S^k}0`M8MqwdWYSG_-h=0{mae*lceQJ)8xXfxm)=%XW~EK&eB#2kAbTSZf;sJu|= z$&}a+@QOo8YsguRmHJ=^6-5LK=K;))|S)?d+Peg)lGh(Co0K59Zq3-y5^%OICAQsls zmyu4^kFjo4GpUCE$@wZO(WDF8J_OWL3I?@GpSL4p+*>^EWAaT3CNOdfMdGs%InJ~t zp6xG60Se(Ov3T`jt3_~-_jirnWZ%@aw#aW|Rf8$ypWAI>>2GStq#-53BogReR8cgP zTN*)6J1gI6F|cY$KiQIxr=oa=n2+)XM-YWkT9WPVB-15qz=AH?f*|x<7~-;#<+tsZ z3mJCq$aR$gJwd_5Y@B{#cQlr7tJ!+@qnx^29^1Yr&(#ZPVX$1TwTK6>fY}_nC}R89 z$A#-8<6Mms^tqoc$GS^S+DZ#1pWrUMltm*9=Vq_dTxZd>AlwcZ(yrJW@+pMoK*a3L zF-Fpa)X<1(3X`y3al2Gn6%ug;=7I;+mGX3QaSRqW^##Gs_7$}xJDGu)3(iLA+~&#F z>QI3jzghc*mNt=d2S~s=u8VZaaCQVC1sptz_%~nruX#G;>8=_qw%+>-tN@ukxpc}- z9s&zcHobd8gY>E2X6*CjdL6{+WM%Elbb-w86&zwEB|<7|S2uA;PdafNr^Xm9%jd|( z$;lGSJ%7TEXHRiGXIs7BdvLJQlD{Xgl}U748=Bv^KHtL~iOOG&Osb^AIN9VdcdRxT z;mwy>$$7MS{jn^1+SU{%@tV9Q#HRIh^FukDd3~hx+UFkhGnii4wN!LfuI1Tq>aowt z(K<9Tt8#{^hAt+o^zL*U+W0EG!F_q?C9zT+ck@$v)u@TSW>o?{)%~UuR;E%LKTD4F zx-rhnxNrF;x7ofwImXI`?J)A#z~x`dl^6Za+$uf=RIb9#IS7+M4#mb&Vkk8EnWFAj zQEg;+mKB2a&UT}p^g6E%RWrYg`K+^hr|@s=37>mTxoXU7_dQo1WzZ_4_1t6Bt4+W& z{c)g*l#)_m;CMpBLscP^1Y7M-{o*gLIH!|s>rVpi8cbwLz6`EW16bA2TamkqwuV9+ z1gu~i+u|I;io)LCDz1&td_q_${P@8cmT zZ_E{~!(#*`IL^QD4}{m@%Xq%BjqThWKAE3)#;d02DHvvDHx3#hdc?;S*}}iZSt&kH zprs8@IfS9-@LcB{Y(qze_e6tn%znCw^K=u&2r7RDv^TjH%$@CKr@0^Qx>qdN!!6c_ z3ypS6zEChpJ>}aN{(^@PBHo3q1tI3(&ehMCu6n$nixkoo$s-Pja7!D@XA-!Auc`Q$ zbm3qrB0jakX6(f{*pKbfbxKk1_C`ZT=|))#=HH-oBvU8VaPW1ug~AqJ#?yJ!j?5Z|20#Thn`TBCP#6OMGRAinmT^KETGXK&&2+)%8{Unhy%A@7HbQ+I3zPRzy;1`&YQ>#&J4ex&u2St@_idQq8BJ38OkpxPRvY8l z2zu|a=E?dyJSA2yD6O~g3B#l5Flp#+xVOD?-8x{%RwJ=q8IVr-*mRYb7FaoP&S+{Q zCKn8Nb!Cw7a_qDLItZl-uS26PJ0rLV>O3Po2P$=IYHwWK2bN`EEi?wn6S^onKU>IH z7%G+sl?AvRCZz-Qp0Zd+HEGmYIod)y;-hYsp2f=?V=}c(mKd0D7p^`&c~D7L{rtOp z5XoXK?Iomkl*Et{E@US%*aVS4X1fWqK@`YOfY3rTWH6RFcmi!^XDk>sEp#&8| zc|ZUC9)xiP@wc&TQ8J3HXQYl@`WX6tK$9?Tf>&#~ZU=M3T0hW0R^HmK3LViMU3kH4 zLn2Wi@rAC}b1NZ=lxy&G1;XH7l|+}5ld~RHuaZmk4!LiN;v}tFvW~^JHW=`0l@E_# zDNbTBP-Z4_mU)|8ViLqqBji2PXCHX0K<2Z{z@W#tU~s|>EvOF!Bf7oQKke>zsA)79 z?(jX|vVUU~zwRK~s>(8o{WNSMlj(l$kVgDT*R@%0i*3g3 z6oq`KwL>@vRx*V(x>qgs)VclrD=Dy~5g(Jyu#sclb)iwNMQJG&C}L0F6gxB@b2{zX zBpbBqPui7B7432mN@xYE^AUqn29nvb9hUG{a7^km-?1`w70cJ&CVNgF7dXrwPb#E6Z8~jex zM^e%d716=`eUOJp3oAHHz{ND1g!i-Dr&n)rP({g%Gy>A+POpg$a~0CQH*wl7WQK$AVh4}(2>=n&7Z}{F%ySiWGC9%AU((dorzb>01_7kGr zR3&wE`T|`%T}<-Q!i%cn^!I3FsxBu9g<2f4xGGQh;5`1r=fXUgFqI-4`&(yGnjPvdJhk z(zyr^|F~kTdPMF|A^JrgThkM|$&(y};qR2D?50$#^0T}k#C#X{(!x0ay=Jc0b!(xZ zb>Ol70CH)CPG`tKuE*qBZ~jS1uzK-VmE)blH{IB2XZjOnWlZs!8v+e5az2Fj4HLVv zfWB%xS^HHA*!pyu+sG4$7%rFVm`vJn#3e0l5kC=Aq-4)A)YJ@aELW42ze3k}qQevfU}U?9L&~h+R~$Je|DNN1k2j&3xKRik+7Iuy}Pi}NPE%E2GE zLV4GJ)y`^(z#P*+`>uWvg{C>0U4007PKyZ(T$;*x>8|rIlTB0<*6lc8t|3pa8&y0X zUzjk>to%?}n8?I(beC|vz#cPQUw^*%W6r*ks%yDtMwz6<~E-dbRQuvar$xe*_oT~ zAifP7WWOhCmnl?!@>&S7=0&*O2b^}_DJ>IR^!xnb8&-chTPHT1Vk-Z$4jJU4y{V?s zXKtF`+0!pK6(975zF7tDOYXbA;IvaC@;HAL!Q;L?S;9nA>l3LxowlBb3FDIRP@^Ye z;@N>Dv81m8`M>VEA%4ihL-idJQhL^Xw=rLU=H8HIzabe#gDu@kPS;jH_uV}eB;F7wN7FXxOU!d>ZC zv(Evu`UFY7MH(O@P2vhv9$3p>Tab zNfFU?C&U8imuF~b%_3?$1bVqPK6M;Jg4$^BZZ^N{*zTmZK!LXB0l*R@U)DkeDCM^P zY_W}RJ2^9`4X8zLjCV+yQy{f;()2`YMFE4&`0Ae*cK*_0?|ogTI&6 zzn3ccr|_MK-&LwEmUrc42t17xl?X-L-b}`n@B2nytm4iG01^e2f&sU^_Ug8Tyt56I zK2kK@{#z`@RFn-Wwb4E=jj&(defh5Ke|dOwULZN^a!n-WRleZz1q0`FZbwva(;Y0| z;`05~k8PL=6=Om?n&oH9>juX-J`-b~z)f_~>m*jAIni_x`=rTIW1a!~#km9)XaC2e z|9PRf@SVC{{MyX6`l)i}7&FA3K6!E#2lIf?@&I|~-M$?bdC)?KnCr51u5XdV+joii za*!;IQQaD5Lw1vntYb@@^T(c)TT|D85ycu>fH5o+;1CxN5NhWr1wHqv9QWHH9cH-$tvICZvUDmL-;_UWUiB{BQMP*Z3@*z=t^zbInl=x@OQ$hO!lQw{H~`^T5j~BX ztk6+=OftiU+xI?sO6(@3IH+O{T;L9M(r9oHm~A!>8_wd6m{WiuM-D<`3tQ~Ad8k`_ohU-l1T_O9Lbjtp*Vu>a-x zapzbb&Z){iNqvrf)SIy6hJBG6WMLX$e;uDv!Et=fXN(J`*qSM+t9OJNtEOM@x?4ly zs)qHft;SA3>iZW*E64jnKv$&iWzW-mgZE@^N@KZ|f)GC67{<@#S7rswe1_l*u^{76 z(Yp-)%Tti~+EjmCgUDKrC3o+;gn)knb1~)#yDV;-p~Okksj}NF4dWYvKhBM1GtPer<$F&idlinuBiY} z1u8QZ86z;@b^ah^z@rA*SlC^tQ(InHp#_SraU$22qz-@#X(}p z>4QpD(t000PkkFt=b+0{nH*x9ZT+4d72CSsG4DS+&qAx1!cjqaCTc?s)Iyw|Z6;fF8% z#v2&RyLT5X2@~0j2J2eeJCAyaE1oqmc@_OUn|H3{-8SYsie$shOyBul!1o$ij7rLw zfDR&ZlYs3K$Sw&AxU^gpgf7)v`|Wcj2ct1EcG>84IMk@Q7PwAWjTb9$&F}B=Itld}Z@>!%wXTO?)lU&}%XWh|49 z(-?G=l%vEpu*cCqXrgU7XM!&{{<7cTn-RUE=auXktSq2URoh=6V7neqV0$_$0YV?u z`06us!n#SY{bGFR4E-ejy$GYFB7U;hIgFZUD6bvc3Jsr;X78X?uElfTV!rI+d5wdu zt-CdoPLQ+nq>=4P=&1(a@V7NlR+J&)rwW35X4>I_xeL`Pl>^23xNqp!Y}C+a)l0nz z4JERVzdcjBHh)c?1Zb-aROd*Q%ex0`~fip8{pT!C~Gg zNK1e==ZhM_KZ-#maSH&uxy;B8_9)=H3{yvOdJ3nL>6@$ozjJ-~66-3MS9rqcr+}mai*t;odeK*L1oi*VmP3_mjB=_>C+H8-s>`RkTQwmEs0;(F!mvuMDyQ;wyk z{s$Mngpy8qLJ%RmD=hcG!hQI$`Vi5(Ak4{)((D0k$D%)KUaK` z5aOu5SDlJ4>fS1+@Td{JXyjjxWTp0(PgK@b8K)#2N)E(4b8UWE{87D5+cDj>P;shM zmS&bB2g#|PD$FW-Ecb{D=u)@sNjC4@lkKzsNHT+>tu}!lR8+34z_MF2`=U&Rn@=4) zxoj8mBa}1PM=ke`FY;bno-~lMG9`81UMcDGIMemTki}EWJ)XY=v9~Pg9u{xrB;oJx zE1Ia@O5VFr&Wi}U1ef1=)`5+;7O5pXzH&^C$i0o|jZW{coY9@vz7u$P_;F#WEUX=R z<#*7b8K_^U4J6+?c6mF{TA+)`?`E_KR5CQ#CGmPV`R8ctOUx&Z$QdK!hC^B85!)A z()+BkS<8`)fw1k$CuzOzW5>HEQoBQhY7;kh&FwzuOkKsrIZotCNma1`$$%I=uHd*N z0-Eb2xWy85dn8Bol(ZdMJAfjpDiCm0?-=?u*i&Sbfj_Wg%T=J<0+B_?4mNs+orjAf1qI5_*f~aDqzt~ z!qVzRi!Z|+U5Cb=R;nk%rFPC;)VZN$6p-?I$bAESfl?n`k5wvMeSUythV#b$_PlsR|SmNCeK2V#1E^nUW&kv6o<*v_;2bd#e6Se3y*) z=$vC}PU1DhZfmN&Y(V=YbOG9s86BCsFF-_Z`TZ5r)-BFXH@izVSHb!As~>x>4uGss z#zE85K+t`?9^&*g%2HLaC%GpK8Sadk$U+Axg_F*v8jbyqm=D%OwCI#x!tVjZH;vQt zEc1&NC|ozavSCJ8H2d>Z)nRIX&gLoKQtl2_C`~aNTU-h1MX2d5D})UW9GY;J1R_F(rKW*gfgivY+Yd) zg6`xpcd}u$e+F9+R=)?lY=x09i%v$AEYSvS;1EYLL7tObmg6GjrY@n;WHf-yq)e7; z_cUSAQdE3wcaV+2n2zOWsuEg0df_ZV1?GERdY`KI%;F?QIP&WX1B>Dv=H zUW`QPX;8oJe6CZW&oV&t4yG|VZ5vv&+@qv>BxC;l$JU~hcBQUac1dC=8qyX6Ztr{C z0Ndv#Bgfnpo9tu!yj9Zez$$r?07f1hc2D8-)PKh+ZzSYd4WDchcE1-&e$5P{LGHRC)caghNJg*^}%#t1i zMi^d-nvZM+cX2^5YFyQC&$gLrp-r>?>rWhS=F#4^F# zX!hnHcLA$0d0M9!IAwbb99bS6qdM8wJa+no!L4qXqXZ1LVRT-E^qvx1O!li~T5<07 zB=22kM;VZ=ZR5B|6JUHTWBXNzL_vHQazdAGrG1hbeqUvRLkLl!C`0WkyMvtR3;?mV z7%iElM)VwtZ3Y63iMJ2vby(0n!f0xBInvmgt7x6%g?9*AtRFyLVll~iHS#!L@11Iv zFauFkqv~Qy?_ho*vr*ej3+3Y6$B-M1pKDL!cX9acuF7n93(}+DxUT0fIW3PZ2u19q zLs^5Shqv6k(|M=g=Ib84@DlZecQY*EbmP1+H#}U#(F1Qk97yM@!hn^obYA+tTI>7i zvov=T78W)%xy4mMG5Nm%O>z>3*S|-$@)~SIie#2*cb+W;2M9oZ2>!UVb}Y{U6v-4M zsHZ16KS$_S+E|rxtkCP1k$|~&cid%NlIFUMa$_=Mp-^jzu9p4>1C} zk#qN^^Wq*&+NDfbm6(qD5)$S~z$5o&0*SWq#ENLs7~rMs0G=vU1^o((?w zEm0g_HtGR(I*f9BflbNg6FVj$#fW+S3}v@&(;L5W6)y%ojw3coT_4SI3`|L#P)`MMwg%B_Qd-Ks*eC#d%M zuQeiR1AMO2HIqxWr^@Dcjjs5lLb|}J|AZ@zJu93ePJtqqgsixjkXEIJ5n#|8?~w6{ zVxD&%C|6Rrj9@(j6<4d+)cHYGJ1b_{5Tf?!${i!5ZHp#qrGmoQZ1pf5HPuXa&UDRR zrHk}DX6*8?@pFE+XLDbK6Gx@!I`jr!8KNViErMn&yqL`bL02QXrF6PlCg?j>(f1qQ zSDAno3SUIT3B|lQ@3m)H>i(DGd5?FmVN`iJh947Z4W=(w@&FZjqlGuXFnbrhHCGHU)qsLM>< zd{7eGW1&t|c0n_JX@Y3tJa~Ve&_`=?wCo4-&Cva2J%N9Jx+DS-IvwkN%0L_;9g-8X z#bV2pWBTt2@|M zs3{^c!*}X;HEFt(n^~m!u^jvZwEMd>6)b*;-9&lO&>i!y|J+Xqel2pF|6Q0N0%7)l;KYldx8HX5%qZv(;ebB{8CB^@Nz+&V1 Optional[str]: @@ -35,7 +49,9 @@ def run_command(cmd: List[str]) -> Optional[str]: -def get_log_streams(instance_id: str, log_group: str = "/aws/ec2/github-runners") -> List[Dict]: +def get_log_streams(instance_id: str, log_group: str = None) -> List[Dict]: + if log_group is None: + log_group = DEFAULT_CLOUDWATCH_LOG_GROUP """Get CloudWatch log streams for an instance.""" cmd = [ "aws", "logs", "describe-log-streams", @@ -76,32 +92,35 @@ def get_log_events(log_group: str, log_stream: str, limit: int = 100, start_from def parse_timestamp(ts_str: str) -> Optional[datetime]: - """Parse various timestamp formats.""" - # Try ISO format first - try: - return datetime.fromisoformat(ts_str.replace("+00:00", "+00:00")) - except: - pass - - # Try log format: "Thu Aug 14 00:29:25 UTC 2025" + """Parse various timestamp formats and ensure timezone is set.""" try: - return datetime.strptime(ts_str, "%a %b %d %H:%M:%S %Z %Y").replace(tzinfo=timezone.utc) + dt = date_parser.parse(ts_str) + # If no timezone info, assume UTC + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt except: - pass - - return None + return None def extract_timestamp_from_log(message: str) -> Optional[datetime]: """Extract timestamp from log message.""" # Pattern: [Thu Aug 14 00:29:25 UTC 2025] - match = re.search(r'\[([^]]+UTC \d{4})]', message) + match = re.search(r'\[([^]]+UTC \d{4})\]', message) + if match: + return parse_timestamp(match.group(1)) + + # Pattern: [2025-08-14 17:37:20] + match = re.search(r'\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]', message) if match: return parse_timestamp(match.group(1)) + return None -def analyze_instance(instance_id: str, log_group: str = "/aws/ec2/github-runners") -> Dict: +def analyze_instance(instance_id: str, log_group: str = None) -> Dict: + if log_group is None: + log_group = DEFAULT_CLOUDWATCH_LOG_GROUP """Analyze runtime and job execution for an instance.""" result = { "instance_id": instance_id, @@ -119,14 +138,18 @@ def analyze_instance(instance_id: str, log_group: str = "/aws/ec2/github-runners # Get CloudWatch logs log_streams = get_log_streams(instance_id, log_group) - # Check if logs are empty (all streams have 0 bytes) - logs_empty = all(stream.get("storedBytes", 0) == 0 for stream in log_streams) if log_streams else True + # Check if logs are empty (all streams have no events) + # Note: storedBytes is often 0 even when there's data, so check for event timestamps instead + logs_empty = all( + stream.get("firstEventTimestamp") is None and stream.get("lastEventTimestamp") is None + for stream in log_streams + ) if log_streams else True # Extract instance info from logs if log_streams and not logs_empty: # Try to get launch time and instance type from runner-setup log for stream in log_streams: - if "/runner-setup" in stream["logStreamName"]: + if f"/{LOG_STREAM_RUNNER_SETUP}" in stream["logStreamName"]: # Get first events for launch time events = get_log_events(log_group, stream["logStreamName"], limit=50, start_from_head=True) @@ -159,6 +182,12 @@ def analyze_instance(instance_id: str, log_group: str = "/aws/ec2/github-runners result["instance_type"] = match.group(1).lower() break + # Look for region in metadata + if "Region:" in msg: + match = re.search(r'Region:\s+(\S+)', msg) + if match: + result["tags"]["Region"] = match.group(1) + # Look for repository name if "Repository:" in msg or "GITHUB_REPOSITORY" in msg: match = re.search(r'Repository:\s+(\S+)|GITHUB_REPOSITORY=(\S+)', msg) @@ -173,14 +202,14 @@ def analyze_instance(instance_id: str, log_group: str = "/aws/ec2/github-runners # Find termination time for stream in log_streams: - if "/termination" in stream["logStreamName"]: + if f"/{LOG_STREAM_TERMINATION}" in stream["logStreamName"]: events = get_log_events(log_group, stream["logStreamName"]) for event in events: - if "proceeding with termination" in event["message"]: + if LOG_MSG_TERMINATION_PROCEEDING in event["message"]: ts = extract_timestamp_from_log(event["message"]) if ts: result["termination_time"] = ts - elif "Runner removed from GitHub successfully" in event["message"]: + elif LOG_MSG_RUNNER_REMOVED in event["message"]: ts = extract_timestamp_from_log(event["message"]) if ts and not result["termination_time"]: result["termination_time"] = ts @@ -204,42 +233,58 @@ def analyze_instance(instance_id: str, log_group: str = "/aws/ec2/github-runners job_ends = {} for stream in log_streams: - if "/job-started" in stream["logStreamName"]: + if f"/{LOG_STREAM_JOB_STARTED}" in stream["logStreamName"]: events = get_log_events(log_group, stream["logStreamName"]) for event in events: msg = event.get("message", "") - # Parse job start events - look for "Job STARTED" pattern - if "Job STARTED" in msg: + # Parse job start events - look for the job started prefix + if LOG_PREFIX_JOB_STARTED in msg or "Job STARTED" in msg: # Extract timestamp ts = extract_timestamp_from_log(msg) if ts: - # Extract job name and run info - # Pattern: "Job STARTED : Test pip install - multiple versions/install (Run: 16952719799/11, Attempt: 1)" - match = re.search(r'Job STARTED\s*:\s*([^(]+)\s*\(Run:\s*(\d+)/(\d+)', msg) + # Extract job name - try both patterns + # Pattern 1: "Job started: job-name" (using LOG_PREFIX_JOB_STARTED) + # Pattern 2: "Job STARTED : Test pip install - multiple versions/install (Run: 16952719799/11, Attempt: 1)" + # Create pattern that handles both cases + prefix_pattern = re.escape(LOG_PREFIX_JOB_STARTED.rstrip(':')) + match = re.search(rf'(?:{prefix_pattern}|Job STARTED)\s*:\s*([^(\n]+?)(?:\s*\(Run:\s*(\d+)/(\d+))?$', msg, re.IGNORECASE) if match: job_name = match.group(1).strip() - run_id = match.group(2) - job_num = match.group(3) - job_key = f"{run_id}/{job_num}" + run_id = match.group(2) if match.group(2) else None + job_num = match.group(3) if match.group(3) else None + + if run_id and job_num: + job_key = f"{run_id}/{job_num}" + else: + # Use job name as key if no run info + job_key = job_name job_starts[job_key] = (ts, job_name) - elif "/job-completed" in stream["logStreamName"]: + elif f"/{LOG_STREAM_JOB_COMPLETED}" in stream["logStreamName"]: events = get_log_events(log_group, stream["logStreamName"]) for event in events: msg = event.get("message", "") - # Parse job completion events - look for "Job COMPLETED" pattern - if "Job COMPLETED" in msg: + # Parse job completion events - look for the job completed prefix + if LOG_PREFIX_JOB_COMPLETED in msg or "Job COMPLETED" in msg: # Extract timestamp ts = extract_timestamp_from_log(msg) if ts: - # Extract job name and run info - # Pattern: "Job COMPLETED: Test pip install - multiple versions/install (Run: 16952719799/11, Attempt: 1)" - match = re.search(r'Job COMPLETED\s*:\s*([^(]+)\s*\(Run:\s*(\d+)/(\d+)', msg) + # Extract job name - try both patterns + # Pattern 1: "Job completed: job-name" (using LOG_PREFIX_JOB_COMPLETED) + # Pattern 2: "Job COMPLETED: Test pip install - multiple versions/install (Run: 16952719799/11, Attempt: 1)" + # Create pattern that handles both cases + prefix_pattern = re.escape(LOG_PREFIX_JOB_COMPLETED.rstrip(':')) + match = re.search(rf'(?:{prefix_pattern}|Job COMPLETED)\s*:\s*([^(\n]+?)(?:\s*\(Run:\s*(\d+)/(\d+))?$', msg, re.IGNORECASE) if match: job_name = match.group(1).strip() - run_id = match.group(2) - job_num = match.group(3) - job_key = f"{run_id}/{job_num}" + run_id = match.group(2) if match.group(2) else None + job_num = match.group(3) if match.group(3) else None + + if run_id and job_num: + job_key = f"{run_id}/{job_num}" + else: + # Use job name as key if no run info + job_key = job_name job_ends[job_key] = (ts, job_name) # Match starts and ends @@ -419,7 +464,11 @@ def get_region_name(region_code: str) -> str: return region_code -def calculate_cost(instance_type: str, runtime_seconds: int, region: str = "us-east-1") -> float: +def calculate_cost( + instance_type: str, + runtime_seconds: int, + region: str = "us-east-1", +) -> float: """Calculate cost based on instance type and runtime.""" hourly_cost = get_instance_price(instance_type, region) if hourly_cost == 0: @@ -533,13 +582,19 @@ def main(): except Exception as e: err(f"Error analyzing {instance_id}: {e}") - # Add failed result + # Add failed result with all required fields results.append({ "instance_id": instance_id, "error": str(e), "total_runtime_seconds": 0, "job_runtime_seconds": 0, - "estimated_cost": 0 + "estimated_cost": 0, + "instance_type": "unknown", + "state": "error", + "launch_time": None, + "termination_time": None, + "jobs": [], + "tags": {} }) # Sort results by instance ID for consistent output diff --git a/scripts/update-snapshots.sh b/scripts/update-snapshots.sh new file mode 100755 index 0000000..5802c79 --- /dev/null +++ b/scripts/update-snapshots.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +pytest --snapshot-update -m 'not slow' +pytest -vvv -m 'not slow' . diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index edd8f77..60d6ad5 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -1,4 +1,14 @@ from ec2_gha.start import StartAWS +from ec2_gha.defaults import ( + EC2_INSTANCE_TYPE, + INSTANCE_COUNT, + INSTANCE_NAME, + MAX_INSTANCE_LIFETIME, + RUNNER_GRACE_PERIOD, + RUNNER_INITIAL_GRACE_PERIOD, + RUNNER_POLL_INTERVAL, + RUNNER_REGISTRATION_TIMEOUT, +) from gha_runner.gh import GitHubInstance from gha_runner.clouddeployment import DeployInstance from gha_runner.helper.input import EnvVarBuilder, check_required @@ -10,8 +20,9 @@ def main(): required = ["GH_PAT", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] # Check that everything exists check_required(env, required) - # Timeout for waiting for runner to register with GitHub (default 5 minutes) - timeout = int(os.environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "300")) + # Timeout for waiting for runner to register with GitHub + timeout_str = os.environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "").strip() + timeout = int(timeout_str) if timeout_str else int(RUNNER_REGISTRATION_TIMEOUT) token = os.environ["GH_PAT"] # Make a copy of environment variables for immutability @@ -51,7 +62,21 @@ def main(): raise Exception("Repo cannot be empty") # Instance count is not a keyword arg for StartAWS, so we remove it - instance_count = params.pop("instance_count") + instance_count = params.pop("instance_count", INSTANCE_COUNT) + + # Apply defaults that weren't set via inputs or vars + params.setdefault("max_instance_lifetime", MAX_INSTANCE_LIFETIME) + params.setdefault("runner_grace_period", RUNNER_GRACE_PERIOD) + params.setdefault("runner_initial_grace_period", RUNNER_INITIAL_GRACE_PERIOD) + params.setdefault("runner_poll_interval", RUNNER_POLL_INTERVAL) + params.setdefault("instance_name", INSTANCE_NAME) + params.setdefault("instance_type", EC2_INSTANCE_TYPE) + params.setdefault("region_name", "us-east-1") # Default AWS region + + # image_id is required - must be provided via input or vars + if not params.get("image_id"): + raise Exception("EC2 AMI ID (ec2_image_id) must be provided via input or vars.EC2_IMAGE_ID") + # home_dir will be set to AUTO in start.py if not provided gh = GitHubInstance(token=token, repo=repo) # This will create a new instance of StartAWS and configure it correctly diff --git a/src/ec2_gha/defaults.py b/src/ec2_gha/defaults.py new file mode 100644 index 0000000..40e9cb5 --- /dev/null +++ b/src/ec2_gha/defaults.py @@ -0,0 +1,20 @@ +"""Default values for ec2-gha configuration.""" + +# Instance lifetime and timing defaults +MAX_INSTANCE_LIFETIME = "360" # 6 hours (in minutes) +RUNNER_GRACE_PERIOD = "60" # 1 minute (in seconds) +RUNNER_INITIAL_GRACE_PERIOD = "180" # 3 minutes (in seconds) +RUNNER_POLL_INTERVAL = "10" # 10 seconds +RUNNER_REGISTRATION_TIMEOUT = "300" # 5 minutes (in seconds) + +# EC2 instance defaults +EC2_INSTANCE_TYPE = "t3.medium" + +# Instance naming default template +INSTANCE_NAME = "$repo/$name#$run_number" + +# Default instance count +INSTANCE_COUNT = 1 + +# Home directory auto-detection sentinel +AUTO = "AUTO" diff --git a/src/ec2_gha/log_constants.py b/src/ec2_gha/log_constants.py new file mode 100644 index 0000000..2e84451 --- /dev/null +++ b/src/ec2_gha/log_constants.py @@ -0,0 +1,24 @@ +""" +Constants for log messages used across ec2-gha components. + +These constants ensure consistency between the runner scripts that generate logs +and the analysis tools that parse them. +""" + +# Log stream names (relative to instance ID prefix) +LOG_STREAM_RUNNER_SETUP = "runner-setup" +LOG_STREAM_JOB_STARTED = "job-started" +LOG_STREAM_JOB_COMPLETED = "job-completed" +LOG_STREAM_TERMINATION = "termination" +LOG_STREAM_RUNNER_DIAG = "runner-diag" + +# Log message prefixes +LOG_PREFIX_JOB_STARTED = "Job started:" +LOG_PREFIX_JOB_COMPLETED = "Job completed:" + +# Termination messages +LOG_MSG_TERMINATION_PROCEEDING = "proceeding with termination" +LOG_MSG_RUNNER_REMOVED = "Runner removed from GitHub successfully" + +# Default CloudWatch log group +DEFAULT_CLOUDWATCH_LOG_GROUP = "/aws/ec2/github-runners" \ No newline at end of file diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 7c54add..d5bee80 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -11,6 +11,8 @@ from gha_runner.helper.workflow_cmds import output from copy import deepcopy +from ec2_gha.defaults import AUTO, RUNNER_REGISTRATION_TIMEOUT + @dataclass class StartAWS(CreateCloudInstance): @@ -18,8 +20,6 @@ class StartAWS(CreateCloudInstance): Parameters ---------- - home_dir : str - The home directory of the user. image_id : str The ID of the AMI to use. instance_type : str @@ -32,6 +32,8 @@ class StartAWS(CreateCloudInstance): CloudWatch Logs group name for streaming runner logs. Defaults to an empty string. gh_runner_tokens : list[str] A list of GitHub runner tokens. Defaults to an empty list. + home_dir : str + The home directory of the user. If not provided, will be inferred from the AMI. iam_instance_profile : str The name of the IAM role to use. Defaults to an empty string. key_name : str @@ -63,13 +65,13 @@ class StartAWS(CreateCloudInstance): """ - home_dir: str image_id: str instance_type: str region_name: str repo: str cloudwatch_logs_group: str = "" gh_runner_tokens: list[str] = field(default_factory=list) + home_dir: str = "" iam_instance_profile: str = "" instance_name: str = "" key_name: str = "" @@ -87,7 +89,7 @@ class StartAWS(CreateCloudInstance): tags: list[dict[str, str]] = field(default_factory=list) userdata: str = "" - def _build_aws_params(self, user_data_params: dict) -> dict: + def _build_aws_params(self, user_data_params: dict, idx: int = None) -> dict: """Build the parameters for the AWS API call. Parameters @@ -163,6 +165,10 @@ def _build_aws_params(self, user_data_params: dict) -> dict: # Get run number template_vars["run_number"] = os.environ.get("GITHUB_RUN_NUMBER", "unknown") + # Add instance index if provided (for multi-instance launches) + if idx is not None: + template_vars["idx"] = str(idx) + # Apply the instance name template from string import Template name_template = Template(self.instance_name) @@ -206,6 +212,16 @@ def _build_user_data(self, **kwargs) -> str: The user data script as a string. """ + # Import log constants to inject into template + from ec2_gha.log_constants import ( + LOG_PREFIX_JOB_STARTED, + LOG_PREFIX_JOB_COMPLETED, + ) + + # Add log constants to the kwargs + kwargs['log_prefix_job_started'] = LOG_PREFIX_JOB_STARTED + kwargs['log_prefix_job_completed'] = LOG_PREFIX_JOB_COMPLETED + template = importlib.resources.files("ec2_gha").joinpath("templates/user-script.sh.templ") with template.open() as f: template_content = f.read() @@ -269,8 +285,6 @@ def create_instances(self) -> dict[str, str]: raise ValueError("No GitHub runner tokens provided, cannot create instances.") if not self.runner_release: raise ValueError("No runner release provided, cannot create instances.") - if not self.home_dir: - raise ValueError("No home directory provided, cannot create instances.") if not self.image_id: raise ValueError("No image ID provided, cannot create instances.") if not self.instance_type: @@ -278,8 +292,12 @@ def create_instances(self) -> dict[str, str]: if not self.region_name: raise ValueError("No region name provided, cannot create instances.") ec2 = boto3.client("ec2", region_name=self.region_name) + + # Use AUTO to let the instance detect its own home directory + if not self.home_dir: + self.home_dir = AUTO id_dict = {} - for token in self.gh_runner_tokens: + for idx, token in enumerate(self.gh_runner_tokens): label = gh.GitHubInstance.generate_random_label() # Combine user labels with the generated runner label labels = f"{self.labels},{label}" if self.labels else label @@ -296,13 +314,14 @@ def create_instances(self) -> dict[str, str]: "runner_grace_period": self.runner_grace_period, "runner_initial_grace_period": self.runner_initial_grace_period, "runner_poll_interval": self.runner_poll_interval, + "runner_registration_timeout": environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "").strip() or RUNNER_REGISTRATION_TIMEOUT, "runner_release": self.runner_release, "script": self.script, "ssh_pubkey": self.ssh_pubkey, "token": token, "userdata": self.userdata, } - params = self._build_aws_params(user_data_params) + params = self._build_aws_params(user_data_params, idx=idx) if self.root_device_size > 0: params = self._modify_root_disk_size(ec2, params) result = ec2.run_instances(**params) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index e05c673..51e5eda 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -1,28 +1,162 @@ #!/bin/bash set -e +# Helper functions +log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } +log_error() { log "ERROR: $$1" >&2; } + +# Function to flush CloudWatch logs before shutdown +flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi +} + +# Create common functions file +cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' +#!/bin/bash +log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } +flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi +} +EOCF + +# Get metadata (IMDSv2 compatible) +get_metadata() { + local path="$$1" + local token=$$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $$token" "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" + fi +} + +logger "EC2-GHA: Starting userdata script" +trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR + +INSTANCE_ID=$$(get_metadata "instance-id") + +terminate_instance() { + local reason="$$1" + log "FATAL: $$reason" + log "Terminating instance $$INSTANCE_ID due to setup failure" + + # Try to remove runner if it was partially configured + if [ -f "$$homedir/config.sh" ] && [ -n "$${RUNNER_TOKEN:-}" ]; then + cd "$$homedir" && ./config.sh remove --token "$${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + shutdown -h now + exit 1 +} + +# Trap errors and ensure termination on failure +trap 'terminate_instance "Setup script failed with error on line $$LINENO"' ERR + +# Set up registration timeout failsafe +REGISTRATION_TIMEOUT="$runner_registration_timeout" +# Validate timeout is a number, default to 300 if not +if ! [[ "$$REGISTRATION_TIMEOUT" =~ ^[0-9]+$$ ]]; then + logger "EC2-GHA: Invalid timeout '$$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 +fi +logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" +( + log "Watchdog: Starting $$REGISTRATION_TIMEOUT second timeout" + sleep $$REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $$REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi +) & +REGISTRATION_WATCHDOG_PID=$$! +log "Watchdog PID: $$REGISTRATION_WATCHDOG_PID" +# Save watchdog PID to file so it survives exec redirect +echo $$REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid $userdata +# Initialize homedir from template variable +homedir="$homedir" + +# Determine home directory if not specified or set to AUTO +if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + # Check for common cloud-init created users + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$$user" &>/dev/null; then + homedir="/home/$$user" + log "Auto-detected: $$homedir" + break + fi + done + + # Fallback if no standard user found + if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then + # Get the first non-root user's home directory that actually exists + homedir=$$(getent passwd | awk -F: '$$3 >= 1000 && $$3 < 65534 && $$6 ~ /^\/home\// {print $$6}' | while read dir; do + if [ -d "$$dir" ]; then + echo "$$dir" + break + fi + done) + + if [ -z "$$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $$homedir" + else + # Use stat to get the actual owner of the directory + owner=$$(stat -c "%U" "$$homedir" 2>/dev/null || stat -f "%Su" "$$homedir" 2>/dev/null) + log "Detected: $$homedir ($$owner)" + fi + fi +else + log "Using: $$homedir" +fi + +# Fetch instance metadata before redirecting output +INSTANCE_TYPE=$$(get_metadata "instance-type") +INSTANCE_ID=$$(get_metadata "instance-id") +REGION=$$(get_metadata "placement/region") +AZ=$$(get_metadata "placement/availability-zone") + +# Redirect runner setup logs to a file for CloudWatch +exec >> /var/log/runner-setup.log 2>&1 +log "Starting runner setup" + +log "Instance metadata: Type=$${INSTANCE_TYPE} ID=$${INSTANCE_ID} Region=$${REGION} AZ=$${AZ}" + # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=$max_instance_lifetime -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Setting up maximum lifetime timeout: $${MAX_LIFETIME_MINUTES} minutes" +log "Setting up maximum lifetime timeout: $${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep $${MAX_LIFETIME_MINUTES}m && echo '[$$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & # Configure CloudWatch Logs if enabled if [ "$cloudwatch_logs_group" != "" ]; then - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Installing CloudWatch agent" + log "Installing CloudWatch agent" # Use a subshell to prevent CloudWatch failures from stopping the entire script ( # Wait for dpkg lock to be released (up to 2 minutes) - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Waiting for dpkg lock to be released..." + log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do if [ $$timeout -le 0 ]; then - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] WARNING: dpkg lock timeout, proceeding anyway" + log "WARNING: dpkg lock timeout, proceeding anyway" break fi - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] dpkg is locked, waiting... ($$timeout seconds remaining)" + log "dpkg is locked, waiting... ($$timeout seconds remaining)" sleep 5 timeout=$$((timeout - 5)) done @@ -67,13 +201,13 @@ if [ "$cloudwatch_logs_group" != "" ]; then "timezone": "UTC" }, { - "file_path": "$homedir/_diag/Runner_**.log", + "file_path": "$$homedir/_diag/Runner_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, { - "file_path": "$homedir/_diag/Worker_**.log", + "file_path": "$$homedir/_diag/Worker_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" @@ -92,179 +226,113 @@ EOF -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ -s - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] CloudWatch agent started" - ) || echo "[$$(date '+%Y-%m-%d %H:%M:%S')] WARNING: CloudWatch agent installation failed, continuing without it" + log "CloudWatch agent started" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi # Configure SSH access if public key provided if [ -n "$ssh_pubkey" ]; then - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Configuring SSH access" + log "Configuring SSH access" # Determine the default user based on the home directory owner - DEFAULT_USER=$$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + DEFAULT_USER=$$(stat -c "%U" "$$homedir" 2>/dev/null || echo "root") # Create .ssh directory if it doesn't exist mkdir -p "$homedir/.ssh" chmod 700 "$homedir/.ssh" # Add the public key to authorized_keys - echo "$ssh_pubkey" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" + echo "$ssh_pubkey" >> "$$homedir/.ssh/authorized_keys" + chmod 600 "$$homedir/.ssh/authorized_keys" # Set proper ownership if [ "$$DEFAULT_USER" != "root" ]; then - chown -R "$$DEFAULT_USER:$$DEFAULT_USER" "$homedir/.ssh" + chown -R "$$DEFAULT_USER:$$DEFAULT_USER" "$$homedir/.ssh" fi - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] SSH key added for user $$DEFAULT_USER" + log "SSH key added for user $$DEFAULT_USER" fi -# Redirect runner setup logs to a file for CloudWatch -exec >> /var/log/runner-setup.log 2>&1 -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] GitHub runner setup starting" -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Working directory: $homedir" -cd "$homedir" +log "Working directory: $$homedir" +cd "$$homedir" + echo "$script" > pre-runner-script.sh -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Running pre-runner script" +log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 -# We will get the latest release from the GitHub API -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Downloading runner from: $runner_release" -curl -L $runner_release -o runner.tar.gz -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Extracting runner" -# `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` +# Detect architecture and download appropriate runner +ARCH=$$(uname -m) +if [ "$$ARCH" = "aarch64" ] || [ "$$ARCH" = "arm64" ]; then + # For ARM, replace x64 with arm64 in the URL + RUNNER_URL=$$(echo "$runner_release" | sed 's/x64/arm64/g') + log "ARM detected, using: $$RUNNER_URL" +else + RUNNER_URL="$runner_release" + log "x64 detected, using: $$RUNNER_URL" +fi +curl -L $$RUNNER_URL -o runner.tar.gz +log "Extracting runner" +# `--no-overwrite-dir` is important, otherwise `$$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$$homedir` tar --no-overwrite-dir -xzf runner.tar.gz -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Setting up job tracking scripts" -# Create minimal job tracking scripts inline +# Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 -echo "[$$(date)] Job STARTED : $${GITHUB_WORKFLOW}/$${GITHUB_JOB} (Run: $${GITHUB_RUN_ID}/$${GITHUB_RUN_NUMBER}, Attempt: $${GITHUB_RUN_ATTEMPT})" -echo " Repository: $${GITHUB_REPOSITORY}" -echo " Runner: $${RUNNER_NAME}" -JOB_TRACK_DIR="/var/run/github-runner-jobs" -mkdir -p "$${JOB_TRACK_DIR}" -echo "{\"job_id\":\"$${GITHUB_JOB}\",\"run_id\":\"$${GITHUB_RUN_ID}\",\"workflow\":\"$${GITHUB_WORKFLOW}\",\"status\":\"running\"}" > "$${JOB_TRACK_DIR}/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job" -# Update activity timestamp +echo "[$$(date)] ${log_prefix_job_started} $${GITHUB_JOB}" +mkdir -p /var/run/github-runner-jobs +echo '{"status":"running"}' > /var/run/github-runner-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job touch /var/run/github-runner-last-activity +# Mark that at least one job has run +touch /var/run/github-runner-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 -echo "[$$(date)] Job COMPLETED: $${GITHUB_WORKFLOW}/$${GITHUB_JOB} (Run: $${GITHUB_RUN_ID}/$${GITHUB_RUN_NUMBER}, Attempt: $${GITHUB_RUN_ATTEMPT})" -echo " Repository: $${GITHUB_REPOSITORY}" -echo " Runner: $${RUNNER_NAME}" -JOB_TRACK_DIR="/var/run/github-runner-jobs" -if [ -f "$${JOB_TRACK_DIR}/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job" ]; then - sed -i 's/"status":"running"/"status":"completed"/' "$${JOB_TRACK_DIR}/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job" -fi -# Count remaining running jobs -RUNNING_JOBS=$$(grep -l '"status":"running"' "$${JOB_TRACK_DIR}"/*.job 2>/dev/null | wc -l || echo 0) -echo " Running jobs remaining: $${RUNNING_JOBS}" -# Update activity timestamp +echo "[$$(date)] ${log_prefix_job_completed} $${GITHUB_JOB}" +rm -f /var/run/github-runner-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job touch /var/run/github-runner-last-activity EOFC cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' #!/bin/bash exec >> /tmp/termination-check.log 2>&1 -echo "[$$(date)] Checking termination conditions" - -ACTIVITY_FILE="/var/run/github-runner-last-activity" -GRACE_PERIOD="$${RUNNER_GRACE_PERIOD:-60}" -INITIAL_GRACE_PERIOD="$${RUNNER_INITIAL_GRACE_PERIOD:-180}" -JOB_TRACK_DIR="/var/run/github-runner-jobs" - -# Check if activity file exists -if [ ! -f "$${ACTIVITY_FILE}" ]; then - echo "[$$(date)] WARNING: No activity file found, creating it now" - touch "$${ACTIVITY_FILE}" -fi - -# Get last activity time and current time -LAST_ACTIVITY=$$(stat -c %Y "$${ACTIVITY_FILE}" 2>/dev/null || echo 0) -NOW=$$(date +%s) -IDLE_TIME=$$((NOW - LAST_ACTIVITY)) -# Check if any jobs have ever run -if ls "$${JOB_TRACK_DIR}"/*.job 2>/dev/null | grep -q .; then - JOBS_HAVE_RUN=true - CURRENT_GRACE_PERIOD="$${GRACE_PERIOD}" -else - JOBS_HAVE_RUN=false - CURRENT_GRACE_PERIOD="$${INITIAL_GRACE_PERIOD}" -fi - -echo "[$$(date)] Last activity: $$(date -d @$${LAST_ACTIVITY} '+%Y-%m-%d %H:%M:%S')" -echo "[$$(date)] Current time: $$(date '+%Y-%m-%d %H:%M:%S')" -echo "[$$(date)] Idle time: $${IDLE_TIME} seconds (grace period: $${CURRENT_GRACE_PERIOD} seconds)" -echo "[$$(date)] Jobs have run: $${JOBS_HAVE_RUN}" - -# Check for running jobs first -RUNNING_JOBS=$$(grep -l '"status":"running"' "$${JOB_TRACK_DIR}"/*.job 2>/dev/null | wc -l || echo 0) -echo "[$$(date)] Running jobs: $${RUNNING_JOBS}" - -# Show status of each job for debugging -echo "[$$(date)] Current job files:" -for job_file in "$${JOB_TRACK_DIR}"/*.job; do - if [ -f "$${job_file}" ]; then - job_status=$$(grep -o '"status":"[^"]*"' "$${job_file}" || echo "unknown") - echo "[$$(date)] $$(basename "$${job_file}"): $${job_status}" - fi -done || echo "[$$(date)] No job files found" - -# Never terminate if jobs are running -if [ "$${RUNNING_JOBS}" -gt 0 ]; then - echo "[$$(date)] Jobs are still running, not checking idle time" -elif [ "$${IDLE_TIME}" -gt "$${CURRENT_GRACE_PERIOD}" ]; then - echo "[$$(date)] No running jobs and no activity for $${IDLE_TIME} seconds, proceeding with termination" - - # Try to remove runner from GitHub first - if [ -f "$homedir/config.sh" ]; then - echo "[$$(date)] Removing runner from GitHub" - cd "$homedir" - # Stop the runner service - RUNNER_PID=$$(pgrep -f "Runner.Listener" | head -1) - if [ -n "$${RUNNER_PID}" ]; then - echo "[$$(date)] Stopping runner PID $${RUNNER_PID}" - kill -INT "$${RUNNER_PID}" 2>/dev/null || true - # Wait for it to stop - for i in {1..10}; do - if ! kill -0 "$${RUNNER_PID}" 2>/dev/null; then - echo "[$$(date)] Runner stopped" - break - fi - sleep 1 - done - fi - - # Remove runner from GitHub - # We need RUNNER_ALLOW_RUNASROOT=1 to remove as root, just like when we configured it - if RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $token; then - echo "[$$(date)] Runner removed from GitHub successfully" - else - echo "[$$(date)] Failed to remove runner from GitHub" - fi +source /usr/local/bin/runner-common-functions.sh +A="/var/run/github-runner-last-activity" +J="/var/run/github-runner-jobs" +H="/var/run/github-runner-has-run-job" +[ ! -f "$$A" ] && touch "$$A" +L=$$(stat -c %Y "$$A" 2>/dev/null || echo 0) +I=$$(($$(date +%s)-L)) +# Use initial grace period only if no job has ever run +if [ -f "$$H" ]; then G=$${RUNNER_GRACE_PERIOD:-60}; else G=$${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi +R=$$(grep -l '"status":"running"' $$J/*.job 2>/dev/null | wc -l || echo 0) +if [ $$R -eq 0 ] && [ $$I -gt $$G ]; then + echo "[$$(date)] Terminating: idle $$I > grace $$G, proceeding with termination" + [ -f "$$homedir/config.sh" ] && cd "$$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + sleep 2 + if [ -f "$$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $token; then + echo "[$$(date)] Runner removed from GitHub successfully" fi - # Flush CloudWatch logs before shutdown - echo "[$$(date)] Flushing CloudWatch logs" - sync - sleep 5 - + flush_cloudwatch_logs sudo shutdown -h now "Runner terminating after idle timeout" else - echo "[$$(date)] Activity detected within $${CURRENT_GRACE_PERIOD} seconds, not terminating" + if [ $$R -gt 0 ]; then + echo "[$$(date)] $$R job(s) still running, not terminating" + else + echo "[$$(date)] No jobs running, idle $$I seconds (grace period: $$G seconds)" + fi fi EOFT -chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh +chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env -echo "RUNNER_HOME=$homedir" >> .env +echo "RUNNER_HOME=$$homedir" >> .env echo "RUNNER_GRACE_PERIOD=$runner_grace_period" >> .env echo "RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" >> .env echo "RUNNER_POLL_INTERVAL=$runner_poll_interval" >> .env @@ -305,8 +373,8 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer # Get instance metadata for descriptive runner name -INSTANCE_ID=$$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id || echo "unknown") -INSTANCE_TYPE=$$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-type || echo "unknown") +INSTANCE_ID=$$(get_metadata "instance-id") +INSTANCE_TYPE=$$(get_metadata "instance-type") # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-$${INSTANCE_ID}" @@ -334,20 +402,48 @@ fi # The $labels variable already contains user labels and the critical runner-xxx label from Python ALL_LABELS="$labels$${METADATA_LABELS}" -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Configuring runner for repo: $repo" -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Runner name: $${RUNNER_NAME}" -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Labels: $${ALL_LABELS}" -./config.sh --url https://github.com/$repo --token $token --labels "$${ALL_LABELS}" --name "$${RUNNER_NAME}" --disableupdate -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Starting runner" +log "Configuring runner for repo: $repo" +log "Runner name: $${RUNNER_NAME}" +log "Labels: $${ALL_LABELS}" +# Export token for error handler +export RUNNER_TOKEN="$token" + +# Attempt to register. If a runner with the same name exists, config.sh will fail +# We use --unattended to prevent interactive prompts +if ! ./config.sh --url https://github.com/$repo --token $token --labels "$${ALL_LABELS}" --name "$${RUNNER_NAME}" --disableupdate --unattended; then + log_error "Failed to register runner '$${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" +fi +log "Runner registered successfully" + +# Mark registration as complete and kill the watchdog +log "Creating registration marker" +touch /var/run/github-runner-registered +ls -la /var/run/github-runner-registered +# Read watchdog PID from file (survives exec redirect) +if [ -f /var/run/github-runner-watchdog.pid ]; then + WATCHDOG_PID=$$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $$WATCHDOG_PID 2>/dev/null; then + kill $$WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $$WATCHDOG_PID)" + else + log "Watchdog process $$WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid +fi + +log "Starting runner" # Create marker file for cleanup service touch /var/run/github-runner-started # Ensure CloudWatch agent can read diagnostic logs # The cwagent user needs to traverse into $homedir to reach _diag # Make $homedir world-executable (but not readable) so cwagent can traverse it -chmod o+x $homedir -echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Made $homedir traversable for CloudWatch agent" +chmod o+x $$homedir +log "Made $$homedir traversable for CloudWatch agent" # Create _diag directory if it doesn't exist -mkdir -p $homedir/_diag +mkdir -p $$homedir/_diag # The _diag files are already world-readable by default, just ensure the directory is too -chmod 755 $homedir/_diag +chmod 755 $$homedir/_diag ./run.sh diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index c844221..93a9bdf 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -1,31 +1,1135 @@ # serializer version: 1 +# name: test_build_aws_params + dict({ + 'IamInstanceProfile': dict({ + 'Name': 'test', + }), + 'ImageId': 'ami-0772db4c976d21e9b', + 'InstanceInitiatedShutdownBehavior': 'terminate', + 'InstanceType': 't2.micro', + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': list([ + 'test', + ]), + 'SubnetId': 'test', + 'TagSpecifications': list([ + dict({ + 'ResourceType': 'instance', + 'Tags': list([ + dict({ + 'Key': 'Name', + 'Value': 'test', + }), + dict({ + 'Key': 'Owner', + 'Value': 'test', + }), + dict({ + 'Key': 'Repository', + 'Value': 'Open-Athena/ec2-gha', + }), + dict({ + 'Key': 'Workflow', + 'Value': 'CI', + }), + dict({ + 'Key': 'URL', + 'Value': 'https://github.com/Open-Athena/ec2-gha/actions/runs/16725250800', + }), + ]), + }), + ]), + 'UserData': ''' + #!/bin/bash + set -e + # Helper functions + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + log_error() { log "ERROR: $1" >&2; } + + # Function to flush CloudWatch logs before shutdown + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + + # Create common functions file + cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' + #!/bin/bash + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + EOCF + + # Get metadata (IMDSv2 compatible) + get_metadata() { + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi + } + + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + + INSTANCE_ID=$(get_metadata "instance-id") + + terminate_instance() { + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" + + # Try to remove runner if it was partially configured + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + shutdown -h now + exit 1 + } + + # Trap errors and ensure termination on failure + trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + + # Set up registration timeout failsafe + REGISTRATION_TIMEOUT="300" + # Validate timeout is a number, default to 300 if not + if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 + fi + logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + ( + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi + ) & + REGISTRATION_WATCHDOG_PID=$! + log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" + # Save watchdog PID to file so it survives exec redirect + echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + + + + # Initialize homedir from template variable + homedir="/home/ec2-user" + + # Determine home directory if not specified or set to AUTO + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + # Check for common cloud-init created users + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + # Fallback if no standard user found + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Get the first non-root user's home directory that actually exists + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + # Use stat to get the actual owner of the directory + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" + fi + fi + else + log "Using: $homedir" + fi + + # Fetch instance metadata before redirecting output + INSTANCE_TYPE=$(get_metadata "instance-type") + INSTANCE_ID=$(get_metadata "instance-id") + REGION=$(get_metadata "placement/region") + AZ=$(get_metadata "placement/availability-zone") + + # Redirect runner setup logs to a file for CloudWatch + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + + log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" + + # Set up maximum lifetime timeout - do this early to ensure cleanup + MAX_LIFETIME_MINUTES=360 + log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" + nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & + + # Configure CloudWatch Logs if enabled + if [ "" != "" ]; then + log "Installing CloudWatch agent" + # Use a subshell to prevent CloudWatch failures from stopping the entire script + ( + + # Wait for dpkg lock to be released (up to 2 minutes) + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done + + # Download and install CloudWatch agent + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + + # Configure CloudWatch agent + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + { + "agent": { + "run_as_user": "cwagent" + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/var/log/runner-setup.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/runner-setup", + "timezone": "UTC" + }, + { + "file_path": "/tmp/job-started-hook.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/job-started", + "timezone": "UTC" + }, + { + "file_path": "/tmp/job-completed-hook.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/job-completed", + "timezone": "UTC" + }, + { + "file_path": "/tmp/termination-check.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/termination", + "timezone": "UTC" + }, + { + "file_path": "$homedir/_diag/Runner_**.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/runner-diag", + "timezone": "UTC" + }, + { + "file_path": "$homedir/_diag/Worker_**.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/worker-diag", + "timezone": "UTC" + } + ] + } + } + } + } + EOF + + # Start CloudWatch agent + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s + + log "CloudWatch agent started" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + fi + + # Configure SSH access if public key provided + if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then + log "Configuring SSH access" + + # Determine the default user based on the home directory owner + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + + # Create .ssh directory if it doesn't exist + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" + + # Add the public key to authorized_keys + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" + + # Set proper ownership + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi + + log "SSH key added for user $DEFAULT_USER" + fi + + log "Working directory: $homedir" + cd "$homedir" + + echo "echo 'Hello, World!'" > pre-runner-script.sh + log "Running pre-runner script" + source pre-runner-script.sh + export RUNNER_ALLOW_RUNASROOT=1 + # Detect architecture and download appropriate runner + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + # For ARM, replace x64 with arm64 in the URL + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" + else + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" + fi + curl -L $RUNNER_URL -o runner.tar.gz + log "Extracting runner" + # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` + tar --no-overwrite-dir -xzf runner.tar.gz + # Create job tracking scripts + cat > /usr/local/bin/job-started-hook.sh << 'EOFS' + #!/bin/bash + exec >> /tmp/job-started-hook.log 2>&1 + echo "[$(date)] Job started: ${GITHUB_JOB}" + mkdir -p /var/run/github-runner-jobs + echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch /var/run/github-runner-last-activity + EOFS + + cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' + #!/bin/bash + exec >> /tmp/job-completed-hook.log 2>&1 + echo "[$(date)] Job completed: ${GITHUB_JOB}" + rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch /var/run/github-runner-last-activity + EOFC + + cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' + #!/bin/bash + exec >> /tmp/termination-check.log 2>&1 + + source /usr/local/bin/runner-common-functions.sh + A="/var/run/github-runner-last-activity" + J="/var/run/github-runner-jobs" + [ ! -f "$A" ] && touch "$A" + L=$(stat -c %Y "$A" 2>/dev/null || echo 0) + I=$(($(date +%s)-L)) + if ls $J/*.job 2>/dev/null | grep -q .; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi + R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) + if [ $R -eq 0 ] && [ $I -gt $G ]; then + echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" + [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + sleep 2 + if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then + echo "[$(date)] Runner removed from GitHub successfully" + fi + + flush_cloudwatch_logs + sudo shutdown -h now "Runner terminating after idle timeout" + else + echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" + fi + EOFT + + chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + + # Set up runner hooks + echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env + echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env + echo "RUNNER_HOME=$homedir" >> .env + echo "RUNNER_GRACE_PERIOD=61" >> .env + echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env + echo "RUNNER_POLL_INTERVAL=11" >> .env + + # Set up job tracking directory + mkdir -p /var/run/github-runner-jobs + + # Create initial activity timestamp + touch /var/run/github-runner-last-activity + + # Set up periodic termination check using systemd + cat > /etc/systemd/system/runner-termination-check.service << 'EOF' + [Unit] + Description=Check GitHub runner termination conditions + After=network.target + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/check-runner-termination.sh + EOF + + cat > /etc/systemd/system/runner-termination-check.timer << EOF + [Unit] + Description=Periodic GitHub runner termination check + Requires=runner-termination-check.service + + [Timer] + OnBootSec=60s + OnUnitActiveSec=11s + + [Install] + WantedBy=timers.target + EOF + + # Enable and start the timer + systemctl daemon-reload + systemctl enable runner-termination-check.timer + systemctl start runner-termination-check.timer + + # Get instance metadata for descriptive runner name + INSTANCE_ID=$(get_metadata "instance-id") + INSTANCE_TYPE=$(get_metadata "instance-type") + + # Create runner name with just the instance ID for uniqueness + RUNNER_NAME="ec2-${INSTANCE_ID}" + + # Build additional labels with metadata for easier correlation + # These will be visible in the GitHub runner management UI + METADATA_LABELS="" + METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" + METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" + + # Add GitHub workflow metadata passed from the action + if [ -n "CI" ]; then + # Replace spaces and special chars in workflow name for label compatibility + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + fi + if [ -n "16725250800" ]; then + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + fi + if [ -n "1" ]; then + METADATA_LABELS="${METADATA_LABELS},run-number:1" + fi + + # Combine provided labels (user + runner-xxx) with instance metadata labels + # The label variable already contains user labels and the critical runner-xxx label from Python + ALL_LABELS="label${METADATA_LABELS}" + + log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" + log "Runner name: ${RUNNER_NAME}" + log "Labels: ${ALL_LABELS}" + # Export token for error handler + export RUNNER_TOKEN="test" + + # Attempt to register. If a runner with the same name exists, config.sh will fail + # We use --unattended to prevent interactive prompts + if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" + fi + log "Runner registered successfully" + + # Mark registration as complete and kill the watchdog + log "Creating registration marker" + touch /var/run/github-runner-registered + ls -la /var/run/github-runner-registered + # Read watchdog PID from file (survives exec redirect) + if [ -f /var/run/github-runner-watchdog.pid ]; then + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid + fi + + log "Starting runner" + # Create marker file for cleanup service + touch /var/run/github-runner-started + # Ensure CloudWatch agent can read diagnostic logs + # The cwagent user needs to traverse into /home/ec2-user to reach _diag + # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir + log "Made $homedir traversable for CloudWatch agent" + # Create _diag directory if it doesn't exist + mkdir -p $homedir/_diag + # The _diag files are already world-readable by default, just ensure the directory is too + chmod 755 $homedir/_diag + ./run.sh + + ''', + }) +# --- +# name: test_build_aws_params_with_idx + dict({ + 'IamInstanceProfile': dict({ + 'Name': 'test', + }), + 'ImageId': 'ami-0772db4c976d21e9b', + 'InstanceInitiatedShutdownBehavior': 'terminate', + 'InstanceType': 't2.micro', + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': list([ + 'test', + ]), + 'SubnetId': 'test', + 'TagSpecifications': list([ + dict({ + 'ResourceType': 'instance', + 'Tags': list([ + dict({ + 'Key': 'Name', + 'Value': 'ec2-gha/test-0#42', + }), + dict({ + 'Key': 'Repository', + 'Value': 'Open-Athena/ec2-gha', + }), + dict({ + 'Key': 'Workflow', + 'Value': 'CI', + }), + dict({ + 'Key': 'URL', + 'Value': 'https://github.com/Open-Athena/ec2-gha/actions/runs/16725250800', + }), + ]), + }), + ]), + 'UserData': ''' + #!/bin/bash + set -e + # Helper functions + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + log_error() { log "ERROR: $1" >&2; } + + # Function to flush CloudWatch logs before shutdown + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + + # Create common functions file + cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' + #!/bin/bash + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + EOCF + + # Get metadata (IMDSv2 compatible) + get_metadata() { + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi + } + + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + + INSTANCE_ID=$(get_metadata "instance-id") + + terminate_instance() { + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" + + # Try to remove runner if it was partially configured + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + shutdown -h now + exit 1 + } + + # Trap errors and ensure termination on failure + trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + + # Set up registration timeout failsafe + REGISTRATION_TIMEOUT="300" + # Validate timeout is a number, default to 300 if not + if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 + fi + logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + ( + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi + ) & + REGISTRATION_WATCHDOG_PID=$! + log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" + # Save watchdog PID to file so it survives exec redirect + echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + + + + # Initialize homedir from template variable + homedir="/home/ec2-user" + + # Determine home directory if not specified or set to AUTO + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + # Check for common cloud-init created users + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + # Fallback if no standard user found + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Get the first non-root user's home directory that actually exists + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + # Use stat to get the actual owner of the directory + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" + fi + fi + else + log "Using: $homedir" + fi + + # Fetch instance metadata before redirecting output + INSTANCE_TYPE=$(get_metadata "instance-type") + INSTANCE_ID=$(get_metadata "instance-id") + REGION=$(get_metadata "placement/region") + AZ=$(get_metadata "placement/availability-zone") + + # Redirect runner setup logs to a file for CloudWatch + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + + log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" + + # Set up maximum lifetime timeout - do this early to ensure cleanup + MAX_LIFETIME_MINUTES=360 + log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" + nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & + + # Configure CloudWatch Logs if enabled + if [ "" != "" ]; then + log "Installing CloudWatch agent" + # Use a subshell to prevent CloudWatch failures from stopping the entire script + ( + + # Wait for dpkg lock to be released (up to 2 minutes) + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done + + # Download and install CloudWatch agent + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + + # Configure CloudWatch agent + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + { + "agent": { + "run_as_user": "cwagent" + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/var/log/runner-setup.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/runner-setup", + "timezone": "UTC" + }, + { + "file_path": "/tmp/job-started-hook.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/job-started", + "timezone": "UTC" + }, + { + "file_path": "/tmp/job-completed-hook.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/job-completed", + "timezone": "UTC" + }, + { + "file_path": "/tmp/termination-check.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/termination", + "timezone": "UTC" + }, + { + "file_path": "$homedir/_diag/Runner_**.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/runner-diag", + "timezone": "UTC" + }, + { + "file_path": "$homedir/_diag/Worker_**.log", + "log_group_name": "", + "log_stream_name": "{instance_id}/worker-diag", + "timezone": "UTC" + } + ] + } + } + } + } + EOF + + # Start CloudWatch agent + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s + + log "CloudWatch agent started" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + fi + + # Configure SSH access if public key provided + if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then + log "Configuring SSH access" + + # Determine the default user based on the home directory owner + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + + # Create .ssh directory if it doesn't exist + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" + + # Add the public key to authorized_keys + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" + + # Set proper ownership + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi + + log "SSH key added for user $DEFAULT_USER" + fi + + log "Working directory: $homedir" + cd "$homedir" + + echo "echo 'Hello, World!'" > pre-runner-script.sh + log "Running pre-runner script" + source pre-runner-script.sh + export RUNNER_ALLOW_RUNASROOT=1 + # Detect architecture and download appropriate runner + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + # For ARM, replace x64 with arm64 in the URL + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" + else + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" + fi + curl -L $RUNNER_URL -o runner.tar.gz + log "Extracting runner" + # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` + tar --no-overwrite-dir -xzf runner.tar.gz + # Create job tracking scripts + cat > /usr/local/bin/job-started-hook.sh << 'EOFS' + #!/bin/bash + exec >> /tmp/job-started-hook.log 2>&1 + echo "[$(date)] Job started: ${GITHUB_JOB}" + mkdir -p /var/run/github-runner-jobs + echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch /var/run/github-runner-last-activity + EOFS + + cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' + #!/bin/bash + exec >> /tmp/job-completed-hook.log 2>&1 + echo "[$(date)] Job completed: ${GITHUB_JOB}" + rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch /var/run/github-runner-last-activity + EOFC + + cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' + #!/bin/bash + exec >> /tmp/termination-check.log 2>&1 + + source /usr/local/bin/runner-common-functions.sh + A="/var/run/github-runner-last-activity" + J="/var/run/github-runner-jobs" + [ ! -f "$A" ] && touch "$A" + L=$(stat -c %Y "$A" 2>/dev/null || echo 0) + I=$(($(date +%s)-L)) + if ls $J/*.job 2>/dev/null | grep -q .; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi + R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) + if [ $R -eq 0 ] && [ $I -gt $G ]; then + echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" + [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + sleep 2 + if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then + echo "[$(date)] Runner removed from GitHub successfully" + fi + + flush_cloudwatch_logs + sudo shutdown -h now "Runner terminating after idle timeout" + else + echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" + fi + EOFT + + chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + + # Set up runner hooks + echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env + echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env + echo "RUNNER_HOME=$homedir" >> .env + echo "RUNNER_GRACE_PERIOD=61" >> .env + echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env + echo "RUNNER_POLL_INTERVAL=11" >> .env + + # Set up job tracking directory + mkdir -p /var/run/github-runner-jobs + + # Create initial activity timestamp + touch /var/run/github-runner-last-activity + + # Set up periodic termination check using systemd + cat > /etc/systemd/system/runner-termination-check.service << 'EOF' + [Unit] + Description=Check GitHub runner termination conditions + After=network.target + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/check-runner-termination.sh + EOF + + cat > /etc/systemd/system/runner-termination-check.timer << EOF + [Unit] + Description=Periodic GitHub runner termination check + Requires=runner-termination-check.service + + [Timer] + OnBootSec=60s + OnUnitActiveSec=11s + + [Install] + WantedBy=timers.target + EOF + + # Enable and start the timer + systemctl daemon-reload + systemctl enable runner-termination-check.timer + systemctl start runner-termination-check.timer + + # Get instance metadata for descriptive runner name + INSTANCE_ID=$(get_metadata "instance-id") + INSTANCE_TYPE=$(get_metadata "instance-type") + + # Create runner name with just the instance ID for uniqueness + RUNNER_NAME="ec2-${INSTANCE_ID}" + + # Build additional labels with metadata for easier correlation + # These will be visible in the GitHub runner management UI + METADATA_LABELS="" + METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" + METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" + + # Add GitHub workflow metadata passed from the action + if [ -n "CI" ]; then + # Replace spaces and special chars in workflow name for label compatibility + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + fi + if [ -n "16725250800" ]; then + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + fi + if [ -n "42" ]; then + METADATA_LABELS="${METADATA_LABELS},run-number:42" + fi + + # Combine provided labels (user + runner-xxx) with instance metadata labels + # The label variable already contains user labels and the critical runner-xxx label from Python + ALL_LABELS="label${METADATA_LABELS}" + + log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" + log "Runner name: ${RUNNER_NAME}" + log "Labels: ${ALL_LABELS}" + # Export token for error handler + export RUNNER_TOKEN="test" + + # Attempt to register. If a runner with the same name exists, config.sh will fail + # We use --unattended to prevent interactive prompts + if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" + fi + log "Runner registered successfully" + + # Mark registration as complete and kill the watchdog + log "Creating registration marker" + touch /var/run/github-runner-registered + ls -la /var/run/github-runner-registered + # Read watchdog PID from file (survives exec redirect) + if [ -f /var/run/github-runner-watchdog.pid ]; then + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid + fi + + log "Starting runner" + # Create marker file for cleanup service + touch /var/run/github-runner-started + # Ensure CloudWatch agent can read diagnostic logs + # The cwagent user needs to traverse into /home/ec2-user to reach _diag + # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir + log "Made $homedir traversable for CloudWatch agent" + # Create _diag directory if it doesn't exist + mkdir -p $homedir/_diag + # The _diag files are already world-readable by default, just ensure the directory is too + chmod 755 $homedir/_diag + ./run.sh + + ''', + }) +# --- # name: test_build_user_data ''' #!/bin/bash set -e + # Helper functions + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + log_error() { log "ERROR: $1" >&2; } + + # Function to flush CloudWatch logs before shutdown + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + + # Create common functions file + cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' + #!/bin/bash + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + EOCF + + # Get metadata (IMDSv2 compatible) + get_metadata() { + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi + } + + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - echo 'custom userdata' + INSTANCE_ID=$(get_metadata "instance-id") + + terminate_instance() { + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" + + # Try to remove runner if it was partially configured + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + shutdown -h now + exit 1 + } + + # Trap errors and ensure termination on failure + trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + + # Set up registration timeout failsafe + REGISTRATION_TIMEOUT="300" + # Validate timeout is a number, default to 300 if not + if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 + fi + logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + ( + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi + ) & + REGISTRATION_WATCHDOG_PID=$! + log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" + # Save watchdog PID to file so it survives exec redirect + echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + + + + # Initialize homedir from template variable + homedir="/home/ec2-user" + + # Determine home directory if not specified or set to AUTO + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + # Check for common cloud-init created users + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + # Fallback if no standard user found + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Get the first non-root user's home directory that actually exists + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + # Use stat to get the actual owner of the directory + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" + fi + fi + else + log "Using: $homedir" + fi + + # Fetch instance metadata before redirecting output + INSTANCE_TYPE=$(get_metadata "instance-type") + INSTANCE_ID=$(get_metadata "instance-id") + REGION=$(get_metadata "placement/region") + AZ=$(get_metadata "placement/availability-zone") + + # Redirect runner setup logs to a file for CloudWatch + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + + log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=360 - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" + log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & # Configure CloudWatch Logs if enabled if [ "" != "" ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Installing CloudWatch agent" + log "Installing CloudWatch agent" # Use a subshell to prevent CloudWatch failures from stopping the entire script ( # Wait for dpkg lock to be released (up to 2 minutes) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Waiting for dpkg lock to be released..." + log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do if [ $timeout -le 0 ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: dpkg lock timeout, proceeding anyway" + log "WARNING: dpkg lock timeout, proceeding anyway" break fi - echo "[$(date '+%Y-%m-%d %H:%M:%S')] dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($timeout seconds remaining)" sleep 5 timeout=$((timeout - 5)) done @@ -70,13 +1174,13 @@ "timezone": "UTC" }, { - "file_path": "/home/test-user/_diag/Runner_**.log", + "file_path": "$homedir/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, { - "file_path": "/home/test-user/_diag/Worker_**.log", + "file_path": "$homedir/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" @@ -95,179 +1199,113 @@ -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ -s - echo "[$(date '+%Y-%m-%d %H:%M:%S')] CloudWatch agent started" - ) || echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: CloudWatch agent installation failed, continuing without it" + log "CloudWatch agent started" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi # Configure SSH access if public key provided if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Configuring SSH access" + log "Configuring SSH access" # Determine the default user based on the home directory owner - DEFAULT_USER=$(stat -c "%U" "/home/test-user" 2>/dev/null || echo "root") + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") # Create .ssh directory if it doesn't exist - mkdir -p "/home/test-user/.ssh" - chmod 700 "/home/test-user/.ssh" + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" # Add the public key to authorized_keys - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "/home/test-user/.ssh/authorized_keys" - chmod 600 "/home/test-user/.ssh/authorized_keys" + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" # Set proper ownership if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "/home/test-user/.ssh" + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" fi - echo "[$(date '+%Y-%m-%d %H:%M:%S')] SSH key added for user $DEFAULT_USER" + log "SSH key added for user $DEFAULT_USER" fi - # Redirect runner setup logs to a file for CloudWatch - exec >> /var/log/runner-setup.log 2>&1 - echo "[$(date '+%Y-%m-%d %H:%M:%S')] GitHub runner setup starting" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Working directory: /home/test-user" - cd "/home/test-user" - echo "echo 'test script'" > pre-runner-script.sh - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Running pre-runner script" + log "Working directory: $homedir" + cd "$homedir" + + echo "echo 'Hello, World!'" > pre-runner-script.sh + log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 - # We will get the latest release from the GitHub API - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Downloading runner from: https://example.com/runner.tar.gz" - curl -L https://example.com/runner.tar.gz -o runner.tar.gz - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Extracting runner" - # `--no-overwrite-dir` is important, otherwise `/home/test-user` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `/home/test-user` + # Detect architecture and download appropriate runner + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + # For ARM, replace x64 with arm64 in the URL + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" + else + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" + fi + curl -L $RUNNER_URL -o runner.tar.gz + log "Extracting runner" + # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` tar --no-overwrite-dir -xzf runner.tar.gz - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Setting up job tracking scripts" - # Create minimal job tracking scripts inline + # Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 - echo "[$(date)] Job STARTED : ${GITHUB_WORKFLOW}/${GITHUB_JOB} (Run: ${GITHUB_RUN_ID}/${GITHUB_RUN_NUMBER}, Attempt: ${GITHUB_RUN_ATTEMPT})" - echo " Repository: ${GITHUB_REPOSITORY}" - echo " Runner: ${RUNNER_NAME}" - JOB_TRACK_DIR="/var/run/github-runner-jobs" - mkdir -p "${JOB_TRACK_DIR}" - echo "{\"job_id\":\"${GITHUB_JOB}\",\"run_id\":\"${GITHUB_RUN_ID}\",\"workflow\":\"${GITHUB_WORKFLOW}\",\"status\":\"running\"}" > "${JOB_TRACK_DIR}/${GITHUB_RUN_ID}-${GITHUB_JOB}.job" - # Update activity timestamp + echo "[$(date)] Job started: ${GITHUB_JOB}" + mkdir -p /var/run/github-runner-jobs + echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job touch /var/run/github-runner-last-activity + # Mark that at least one job has run + touch /var/run/github-runner-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 - echo "[$(date)] Job COMPLETED: ${GITHUB_WORKFLOW}/${GITHUB_JOB} (Run: ${GITHUB_RUN_ID}/${GITHUB_RUN_NUMBER}, Attempt: ${GITHUB_RUN_ATTEMPT})" - echo " Repository: ${GITHUB_REPOSITORY}" - echo " Runner: ${RUNNER_NAME}" - JOB_TRACK_DIR="/var/run/github-runner-jobs" - if [ -f "${JOB_TRACK_DIR}/${GITHUB_RUN_ID}-${GITHUB_JOB}.job" ]; then - sed -i 's/"status":"running"/"status":"completed"/' "${JOB_TRACK_DIR}/${GITHUB_RUN_ID}-${GITHUB_JOB}.job" - fi - # Count remaining running jobs - RUNNING_JOBS=$(grep -l '"status":"running"' "${JOB_TRACK_DIR}"/*.job 2>/dev/null | wc -l || echo 0) - echo " Running jobs remaining: ${RUNNING_JOBS}" - # Update activity timestamp + echo "[$(date)] Job completed: ${GITHUB_JOB}" + rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job touch /var/run/github-runner-last-activity EOFC cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - echo "[$(date)] Checking termination conditions" - ACTIVITY_FILE="/var/run/github-runner-last-activity" - GRACE_PERIOD="${RUNNER_GRACE_PERIOD:-60}" - INITIAL_GRACE_PERIOD="${RUNNER_INITIAL_GRACE_PERIOD:-180}" - JOB_TRACK_DIR="/var/run/github-runner-jobs" - - # Check if activity file exists - if [ ! -f "${ACTIVITY_FILE}" ]; then - echo "[$(date)] WARNING: No activity file found, creating it now" - touch "${ACTIVITY_FILE}" - fi - - # Get last activity time and current time - LAST_ACTIVITY=$(stat -c %Y "${ACTIVITY_FILE}" 2>/dev/null || echo 0) - NOW=$(date +%s) - IDLE_TIME=$((NOW - LAST_ACTIVITY)) - - # Check if any jobs have ever run - if ls "${JOB_TRACK_DIR}"/*.job 2>/dev/null | grep -q .; then - JOBS_HAVE_RUN=true - CURRENT_GRACE_PERIOD="${GRACE_PERIOD}" - else - JOBS_HAVE_RUN=false - CURRENT_GRACE_PERIOD="${INITIAL_GRACE_PERIOD}" - fi - - echo "[$(date)] Last activity: $(date -d @${LAST_ACTIVITY} '+%Y-%m-%d %H:%M:%S')" - echo "[$(date)] Current time: $(date '+%Y-%m-%d %H:%M:%S')" - echo "[$(date)] Idle time: ${IDLE_TIME} seconds (grace period: ${CURRENT_GRACE_PERIOD} seconds)" - echo "[$(date)] Jobs have run: ${JOBS_HAVE_RUN}" - - # Check for running jobs first - RUNNING_JOBS=$(grep -l '"status":"running"' "${JOB_TRACK_DIR}"/*.job 2>/dev/null | wc -l || echo 0) - echo "[$(date)] Running jobs: ${RUNNING_JOBS}" - - # Show status of each job for debugging - echo "[$(date)] Current job files:" - for job_file in "${JOB_TRACK_DIR}"/*.job; do - if [ -f "${job_file}" ]; then - job_status=$(grep -o '"status":"[^"]*"' "${job_file}" || echo "unknown") - echo "[$(date)] $(basename "${job_file}"): ${job_status}" - fi - done || echo "[$(date)] No job files found" - - # Never terminate if jobs are running - if [ "${RUNNING_JOBS}" -gt 0 ]; then - echo "[$(date)] Jobs are still running, not checking idle time" - elif [ "${IDLE_TIME}" -gt "${CURRENT_GRACE_PERIOD}" ]; then - echo "[$(date)] No running jobs and no activity for ${IDLE_TIME} seconds, proceeding with termination" - - # Try to remove runner from GitHub first - if [ -f "/home/test-user/config.sh" ]; then - echo "[$(date)] Removing runner from GitHub" - cd "/home/test-user" - # Stop the runner service - RUNNER_PID=$(pgrep -f "Runner.Listener" | head -1) - if [ -n "${RUNNER_PID}" ]; then - echo "[$(date)] Stopping runner PID ${RUNNER_PID}" - kill -INT "${RUNNER_PID}" 2>/dev/null || true - # Wait for it to stop - for i in {1..10}; do - if ! kill -0 "${RUNNER_PID}" 2>/dev/null; then - echo "[$(date)] Runner stopped" - break - fi - sleep 1 - done - fi - - # Remove runner from GitHub - # We need RUNNER_ALLOW_RUNASROOT=1 to remove as root, just like when we configured it - if RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test-token-xyz; then - echo "[$(date)] Runner removed from GitHub successfully" - else - echo "[$(date)] Failed to remove runner from GitHub" - fi + source /usr/local/bin/runner-common-functions.sh + A="/var/run/github-runner-last-activity" + J="/var/run/github-runner-jobs" + H="/var/run/github-runner-has-run-job" + [ ! -f "$A" ] && touch "$A" + L=$(stat -c %Y "$A" 2>/dev/null || echo 0) + I=$(($(date +%s)-L)) + # Use initial grace period only if no job has ever run + if [ -f "$H" ]; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi + R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) + if [ $R -eq 0 ] && [ $I -gt $G ]; then + echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" + [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + sleep 2 + if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then + echo "[$(date)] Runner removed from GitHub successfully" fi - # Flush CloudWatch logs before shutdown - echo "[$(date)] Flushing CloudWatch logs" - sync - sleep 5 - + flush_cloudwatch_logs sudo shutdown -h now "Runner terminating after idle timeout" else - echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" + if [ $R -gt 0 ]; then + echo "[$(date)] $R job(s) still running, not terminating" + else + echo "[$(date)] No jobs running, idle $I seconds (grace period: $G seconds)" + fi fi EOFT - chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env - echo "RUNNER_HOME=/home/test-user" >> .env + echo "RUNNER_HOME=$homedir" >> .env echo "RUNNER_GRACE_PERIOD=61" >> .env echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env echo "RUNNER_POLL_INTERVAL=11" >> .env @@ -308,8 +1346,8 @@ systemctl start runner-termination-check.timer # Get instance metadata for descriptive runner name - INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id || echo "unknown") - INSTANCE_TYPE=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-type || echo "unknown") + INSTANCE_ID=$(get_metadata "instance-id") + INSTANCE_TYPE=$(get_metadata "instance-type") # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-${INSTANCE_ID}" @@ -321,38 +1359,66 @@ METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" # Add GitHub workflow metadata passed from the action - if [ -n "test-workflow" ]; then + if [ -n "CI" ]; then # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$(echo "test-workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi - if [ -n "123456789" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:123456789" + if [ -n "16725250800" ]; then + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" fi if [ -n "42" ]; then METADATA_LABELS="${METADATA_LABELS},run-number:42" fi # Combine provided labels (user + runner-xxx) with instance metadata labels - # The test-label variable already contains user labels and the critical runner-xxx label from Python - ALL_LABELS="test-label${METADATA_LABELS}" - - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Configuring runner for repo: test-org/test-repo" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Runner name: ${RUNNER_NAME}" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Labels: ${ALL_LABELS}" - ./config.sh --url https://github.com/test-org/test-repo --token test-token-xyz --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting runner" + # The label variable already contains user labels and the critical runner-xxx label from Python + ALL_LABELS="label${METADATA_LABELS}" + + log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" + log "Runner name: ${RUNNER_NAME}" + log "Labels: ${ALL_LABELS}" + # Export token for error handler + export RUNNER_TOKEN="test" + + # Attempt to register. If a runner with the same name exists, config.sh will fail + # We use --unattended to prevent interactive prompts + if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" + fi + log "Runner registered successfully" + + # Mark registration as complete and kill the watchdog + log "Creating registration marker" + touch /var/run/github-runner-registered + ls -la /var/run/github-runner-registered + # Read watchdog PID from file (survives exec redirect) + if [ -f /var/run/github-runner-watchdog.pid ]; then + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid + fi + + log "Starting runner" # Create marker file for cleanup service touch /var/run/github-runner-started # Ensure CloudWatch agent can read diagnostic logs - # The cwagent user needs to traverse into /home/test-user to reach _diag - # Make /home/test-user world-executable (but not readable) so cwagent can traverse it - chmod o+x /home/test-user - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Made /home/test-user traversable for CloudWatch agent" + # The cwagent user needs to traverse into /home/ec2-user to reach _diag + # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir + log "Made $homedir traversable for CloudWatch agent" # Create _diag directory if it doesn't exist - mkdir -p /home/test-user/_diag + mkdir -p $homedir/_diag # The _diag files are already world-readable by default, just ensure the directory is too - chmod 755 /home/test-user/_diag + chmod 755 $homedir/_diag ./run.sh ''' @@ -361,29 +1427,163 @@ ''' #!/bin/bash set -e + # Helper functions + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + log_error() { log "ERROR: $1" >&2; } + + # Function to flush CloudWatch logs before shutdown + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + # Create common functions file + cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' + #!/bin/bash + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } + flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ + -a stop -m ec2 2>/dev/null || true + fi + } + EOCF + + # Get metadata (IMDSv2 compatible) + get_metadata() { + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi + } + + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + + INSTANCE_ID=$(get_metadata "instance-id") + + terminate_instance() { + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" + # Try to remove runner if it was partially configured + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + shutdown -h now + exit 1 + } + + # Trap errors and ensure termination on failure + trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + + # Set up registration timeout failsafe + REGISTRATION_TIMEOUT="300" + # Validate timeout is a number, default to 300 if not + if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 + fi + logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + ( + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi + ) & + REGISTRATION_WATCHDOG_PID=$! + log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" + # Save watchdog PID to file so it survives exec redirect + echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + + + + # Initialize homedir from template variable + homedir="/home/ec2-user" + + # Determine home directory if not specified or set to AUTO + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + # Check for common cloud-init created users + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + # Fallback if no standard user found + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Get the first non-root user's home directory that actually exists + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + # Use stat to get the actual owner of the directory + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" + fi + fi + else + log "Using: $homedir" + fi + + # Fetch instance metadata before redirecting output + INSTANCE_TYPE=$(get_metadata "instance-type") + INSTANCE_ID=$(get_metadata "instance-id") + REGION=$(get_metadata "placement/region") + AZ=$(get_metadata "placement/availability-zone") + + # Redirect runner setup logs to a file for CloudWatch + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + + log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=360 - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" + log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & # Configure CloudWatch Logs if enabled if [ "/aws/ec2/github-runners" != "" ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Installing CloudWatch agent" + log "Installing CloudWatch agent" # Use a subshell to prevent CloudWatch failures from stopping the entire script ( # Wait for dpkg lock to be released (up to 2 minutes) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Waiting for dpkg lock to be released..." + log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do if [ $timeout -le 0 ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: dpkg lock timeout, proceeding anyway" + log "WARNING: dpkg lock timeout, proceeding anyway" break fi - echo "[$(date '+%Y-%m-%d %H:%M:%S')] dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($timeout seconds remaining)" sleep 5 timeout=$((timeout - 5)) done @@ -428,13 +1628,13 @@ "timezone": "UTC" }, { - "file_path": "/home/test-user/_diag/Runner_**.log", + "file_path": "$homedir/_diag/Runner_**.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, { - "file_path": "/home/test-user/_diag/Worker_**.log", + "file_path": "$homedir/_diag/Worker_**.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" @@ -453,182 +1653,108 @@ -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ -s - echo "[$(date '+%Y-%m-%d %H:%M:%S')] CloudWatch agent started" - ) || echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: CloudWatch agent installation failed, continuing without it" + log "CloudWatch agent started" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi # Configure SSH access if public key provided if [ -n "" ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Configuring SSH access" + log "Configuring SSH access" # Determine the default user based on the home directory owner - DEFAULT_USER=$(stat -c "%U" "/home/test-user" 2>/dev/null || echo "root") + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") # Create .ssh directory if it doesn't exist - mkdir -p "/home/test-user/.ssh" - chmod 700 "/home/test-user/.ssh" + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" # Add the public key to authorized_keys - echo "" >> "/home/test-user/.ssh/authorized_keys" - chmod 600 "/home/test-user/.ssh/authorized_keys" + echo "" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" # Set proper ownership if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "/home/test-user/.ssh" + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" fi - echo "[$(date '+%Y-%m-%d %H:%M:%S')] SSH key added for user $DEFAULT_USER" + log "SSH key added for user $DEFAULT_USER" fi - # Redirect runner setup logs to a file for CloudWatch - exec >> /var/log/runner-setup.log 2>&1 - echo "[$(date '+%Y-%m-%d %H:%M:%S')] GitHub runner setup starting" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Working directory: /home/test-user" - cd "/home/test-user" - echo "echo 'test script'" > pre-runner-script.sh - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Running pre-runner script" + log "Working directory: $homedir" + cd "$homedir" + + echo "echo 'Hello, World!'" > pre-runner-script.sh + log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 - # We will get the latest release from the GitHub API - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Downloading runner from: https://example.com/runner.tar.gz" - curl -L https://example.com/runner.tar.gz -o runner.tar.gz - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Extracting runner" - # `--no-overwrite-dir` is important, otherwise `/home/test-user` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `/home/test-user` + # Detect architecture and download appropriate runner + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + # For ARM, replace x64 with arm64 in the URL + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" + else + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" + fi + curl -L $RUNNER_URL -o runner.tar.gz + log "Extracting runner" + # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` tar --no-overwrite-dir -xzf runner.tar.gz - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Setting up job tracking scripts" - # Create minimal job tracking scripts inline + # Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 - echo "[$(date)] Job STARTED : ${GITHUB_WORKFLOW}/${GITHUB_JOB} (Run: ${GITHUB_RUN_ID}/${GITHUB_RUN_NUMBER}, Attempt: ${GITHUB_RUN_ATTEMPT})" - echo " Repository: ${GITHUB_REPOSITORY}" - echo " Runner: ${RUNNER_NAME}" - JOB_TRACK_DIR="/var/run/github-runner-jobs" - mkdir -p "${JOB_TRACK_DIR}" - echo "{\"job_id\":\"${GITHUB_JOB}\",\"run_id\":\"${GITHUB_RUN_ID}\",\"workflow\":\"${GITHUB_WORKFLOW}\",\"status\":\"running\"}" > "${JOB_TRACK_DIR}/${GITHUB_RUN_ID}-${GITHUB_JOB}.job" - # Update activity timestamp + echo "[$(date)] Job started: ${GITHUB_JOB}" + mkdir -p /var/run/github-runner-jobs + echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job touch /var/run/github-runner-last-activity EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 - echo "[$(date)] Job COMPLETED: ${GITHUB_WORKFLOW}/${GITHUB_JOB} (Run: ${GITHUB_RUN_ID}/${GITHUB_RUN_NUMBER}, Attempt: ${GITHUB_RUN_ATTEMPT})" - echo " Repository: ${GITHUB_REPOSITORY}" - echo " Runner: ${RUNNER_NAME}" - JOB_TRACK_DIR="/var/run/github-runner-jobs" - if [ -f "${JOB_TRACK_DIR}/${GITHUB_RUN_ID}-${GITHUB_JOB}.job" ]; then - sed -i 's/"status":"running"/"status":"completed"/' "${JOB_TRACK_DIR}/${GITHUB_RUN_ID}-${GITHUB_JOB}.job" - fi - # Count remaining running jobs - RUNNING_JOBS=$(grep -l '"status":"running"' "${JOB_TRACK_DIR}"/*.job 2>/dev/null | wc -l || echo 0) - echo " Running jobs remaining: ${RUNNING_JOBS}" - # Update activity timestamp + echo "[$(date)] Job completed: ${GITHUB_JOB}" + rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job touch /var/run/github-runner-last-activity EOFC cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - echo "[$(date)] Checking termination conditions" - - ACTIVITY_FILE="/var/run/github-runner-last-activity" - GRACE_PERIOD="${RUNNER_GRACE_PERIOD:-60}" - INITIAL_GRACE_PERIOD="${RUNNER_INITIAL_GRACE_PERIOD:-180}" - JOB_TRACK_DIR="/var/run/github-runner-jobs" - - # Check if activity file exists - if [ ! -f "${ACTIVITY_FILE}" ]; then - echo "[$(date)] WARNING: No activity file found, creating it now" - touch "${ACTIVITY_FILE}" - fi - - # Get last activity time and current time - LAST_ACTIVITY=$(stat -c %Y "${ACTIVITY_FILE}" 2>/dev/null || echo 0) - NOW=$(date +%s) - IDLE_TIME=$((NOW - LAST_ACTIVITY)) - - # Check if any jobs have ever run - if ls "${JOB_TRACK_DIR}"/*.job 2>/dev/null | grep -q .; then - JOBS_HAVE_RUN=true - CURRENT_GRACE_PERIOD="${GRACE_PERIOD}" - else - JOBS_HAVE_RUN=false - CURRENT_GRACE_PERIOD="${INITIAL_GRACE_PERIOD}" - fi - - echo "[$(date)] Last activity: $(date -d @${LAST_ACTIVITY} '+%Y-%m-%d %H:%M:%S')" - echo "[$(date)] Current time: $(date '+%Y-%m-%d %H:%M:%S')" - echo "[$(date)] Idle time: ${IDLE_TIME} seconds (grace period: ${CURRENT_GRACE_PERIOD} seconds)" - echo "[$(date)] Jobs have run: ${JOBS_HAVE_RUN}" - - # Check for running jobs first - RUNNING_JOBS=$(grep -l '"status":"running"' "${JOB_TRACK_DIR}"/*.job 2>/dev/null | wc -l || echo 0) - echo "[$(date)] Running jobs: ${RUNNING_JOBS}" - - # Show status of each job for debugging - echo "[$(date)] Current job files:" - for job_file in "${JOB_TRACK_DIR}"/*.job; do - if [ -f "${job_file}" ]; then - job_status=$(grep -o '"status":"[^"]*"' "${job_file}" || echo "unknown") - echo "[$(date)] $(basename "${job_file}"): ${job_status}" - fi - done || echo "[$(date)] No job files found" - - # Never terminate if jobs are running - if [ "${RUNNING_JOBS}" -gt 0 ]; then - echo "[$(date)] Jobs are still running, not checking idle time" - elif [ "${IDLE_TIME}" -gt "${CURRENT_GRACE_PERIOD}" ]; then - echo "[$(date)] No running jobs and no activity for ${IDLE_TIME} seconds, proceeding with termination" - - # Try to remove runner from GitHub first - if [ -f "/home/test-user/config.sh" ]; then - echo "[$(date)] Removing runner from GitHub" - cd "/home/test-user" - # Stop the runner service - RUNNER_PID=$(pgrep -f "Runner.Listener" | head -1) - if [ -n "${RUNNER_PID}" ]; then - echo "[$(date)] Stopping runner PID ${RUNNER_PID}" - kill -INT "${RUNNER_PID}" 2>/dev/null || true - # Wait for it to stop - for i in {1..10}; do - if ! kill -0 "${RUNNER_PID}" 2>/dev/null; then - echo "[$(date)] Runner stopped" - break - fi - sleep 1 - done - fi - # Remove runner from GitHub - # We need RUNNER_ALLOW_RUNASROOT=1 to remove as root, just like when we configured it - if RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test-token-xyz; then - echo "[$(date)] Runner removed from GitHub successfully" - else - echo "[$(date)] Failed to remove runner from GitHub" - fi + source /usr/local/bin/runner-common-functions.sh + A="/var/run/github-runner-last-activity" + J="/var/run/github-runner-jobs" + [ ! -f "$A" ] && touch "$A" + L=$(stat -c %Y "$A" 2>/dev/null || echo 0) + I=$(($(date +%s)-L)) + if ls $J/*.job 2>/dev/null | grep -q .; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi + R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) + if [ $R -eq 0 ] && [ $I -gt $G ]; then + echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" + [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + sleep 2 + if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then + echo "[$(date)] Runner removed from GitHub successfully" fi - # Flush CloudWatch logs before shutdown - echo "[$(date)] Flushing CloudWatch logs" - sync - sleep 5 - + flush_cloudwatch_logs sudo shutdown -h now "Runner terminating after idle timeout" else echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" fi EOFT - chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env - echo "RUNNER_HOME=/home/test-user" >> .env - echo "RUNNER_GRACE_PERIOD=60" >> .env - echo "RUNNER_INITIAL_GRACE_PERIOD=180" >> .env - echo "RUNNER_POLL_INTERVAL=10" >> .env + echo "RUNNER_HOME=$homedir" >> .env + echo "RUNNER_GRACE_PERIOD=61" >> .env + echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env + echo "RUNNER_POLL_INTERVAL=11" >> .env # Set up job tracking directory mkdir -p /var/run/github-runner-jobs @@ -654,7 +1780,7 @@ [Timer] OnBootSec=60s - OnUnitActiveSec=10s + OnUnitActiveSec=11s [Install] WantedBy=timers.target @@ -666,8 +1792,8 @@ systemctl start runner-termination-check.timer # Get instance metadata for descriptive runner name - INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id || echo "unknown") - INSTANCE_TYPE=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-type || echo "unknown") + INSTANCE_ID=$(get_metadata "instance-id") + INSTANCE_TYPE=$(get_metadata "instance-type") # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-${INSTANCE_ID}" @@ -679,38 +1805,66 @@ METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" # Add GitHub workflow metadata passed from the action - if [ -n "test-workflow" ]; then + if [ -n "CI" ]; then # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$(echo "test-workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi - if [ -n "123456789" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:123456789" + if [ -n "16725250800" ]; then + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" fi if [ -n "42" ]; then METADATA_LABELS="${METADATA_LABELS},run-number:42" fi # Combine provided labels (user + runner-xxx) with instance metadata labels - # The test-label variable already contains user labels and the critical runner-xxx label from Python - ALL_LABELS="test-label${METADATA_LABELS}" - - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Configuring runner for repo: test-org/test-repo" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Runner name: ${RUNNER_NAME}" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Labels: ${ALL_LABELS}" - ./config.sh --url https://github.com/test-org/test-repo --token test-token-xyz --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting runner" + # The label variable already contains user labels and the critical runner-xxx label from Python + ALL_LABELS="label${METADATA_LABELS}" + + log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" + log "Runner name: ${RUNNER_NAME}" + log "Labels: ${ALL_LABELS}" + # Export token for error handler + export RUNNER_TOKEN="test" + + # Attempt to register. If a runner with the same name exists, config.sh will fail + # We use --unattended to prevent interactive prompts + if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" + fi + log "Runner registered successfully" + + # Mark registration as complete and kill the watchdog + log "Creating registration marker" + touch /var/run/github-runner-registered + ls -la /var/run/github-runner-registered + # Read watchdog PID from file (survives exec redirect) + if [ -f /var/run/github-runner-watchdog.pid ]; then + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid + fi + + log "Starting runner" # Create marker file for cleanup service touch /var/run/github-runner-started # Ensure CloudWatch agent can read diagnostic logs - # The cwagent user needs to traverse into /home/test-user to reach _diag - # Make /home/test-user world-executable (but not readable) so cwagent can traverse it - chmod o+x /home/test-user - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Made /home/test-user traversable for CloudWatch agent" + # The cwagent user needs to traverse into /home/ec2-user to reach _diag + # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir + log "Made $homedir traversable for CloudWatch agent" # Create _diag directory if it doesn't exist - mkdir -p /home/test-user/_diag + mkdir -p $homedir/_diag # The _diag files are already world-readable by default, just ensure the directory is too - chmod 755 /home/test-user/_diag + chmod 755 $homedir/_diag ./run.sh ''' diff --git a/tests/test_start.py b/tests/test_start.py index 217ca1b..da54f62 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -5,83 +5,71 @@ from moto import mock_aws from ec2_gha.start import StartAWS +from ec2_gha.defaults import AUTO @pytest.fixture(scope="function") -def aws(): +def base_aws_params(): + """Base parameters for StartAWS initialization""" + return { + "gh_runner_tokens": ["testing"], + "home_dir": "/home/ec2-user", + "image_id": "ami-0772db4c976d21e9b", + "instance_type": "t2.micro", + "region_name": "us-east-1", + "repo": "omsf-eco-infra/awsinfratesting", + "runner_grace_period": "120", + "runner_release": "testing", + } + + +@pytest.fixture(scope="function") +def aws(base_aws_params): with mock_aws(): - params = { - "gh_runner_tokens": ["testing"], - "home_dir": "/home/ec2-user", - "image_id": "ami-0772db4c976d21e9b", - "instance_type": "t2.micro", - "region_name": "us-east-1", - "repo": "omsf-eco-infra/awsinfratesting", - "runner_grace_period": "120", - "runner_release": "testing", - } - yield StartAWS(**params) + yield StartAWS(**base_aws_params) -def test_build_user_data(aws, snapshot): - """Test that template parameters are correctly substituted using snapshot testing""" - params = { +@pytest.fixture(scope="function") +def aws_params_user_data(): + """User data params for AWS params tests""" + return { "cloudwatch_logs_group": "", # Empty = disabled - "github_run_id": "123456789", + "github_run_id": "16725250800", "github_run_number": "42", - "github_workflow": "test-workflow", - "homedir": "/home/test-user", - "labels": "test-label", + "github_workflow": "CI", + "homedir": "/home/ec2-user", + "labels": "label", "max_instance_lifetime": "360", - "repo": "test-org/test-repo", + "repo": "omsf-eco-infra/awsinfratesting", "runner_grace_period": "61", "runner_initial_grace_period": "181", "runner_poll_interval": "11", - "runner_release": "https://example.com/runner.tar.gz", - "script": "echo 'test script'", + "runner_release": "test.tar.gz", + "runner_registration_timeout": "300", + "script": "echo 'Hello, World!'", "ssh_pubkey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host", - "token": "test-token-xyz", - "userdata": "echo 'custom userdata'", + "token": "test", + "userdata": "", } - user_data = aws._build_user_data(**params) - # Verify all substitutions happened (no template variables remain) - template_vars = [ f'${k}' for k in params ] - for var in template_vars: - assert var not in user_data, f"Template variable {var} was not substituted" - # Use snapshot to verify the entire output +def test_build_user_data(aws, aws_params_user_data, snapshot): + """Test that template parameters are correctly substituted using snapshot testing""" + user_data = aws._build_user_data(**aws_params_user_data) assert user_data == snapshot -def test_build_user_data_with_cloudwatch(aws, snapshot): +def test_build_user_data_with_cloudwatch(aws, aws_params_user_data, snapshot): """Test user data with CloudWatch Logs enabled using snapshot testing""" - params = { + params = aws_params_user_data | { "cloudwatch_logs_group": "/aws/ec2/github-runners", - "github_run_id": "123456789", - "github_run_number": "42", - "github_workflow": "test-workflow", - "homedir": "/home/test-user", - "labels": "test-label", - "max_instance_lifetime": "360", - "repo": "test-org/test-repo", - "runner_grace_period": "60", - "runner_initial_grace_period": "180", - "runner_poll_interval": "10", - "runner_release": "https://example.com/runner.tar.gz", - "script": "echo 'test script'", + "runner_grace_period": "61", + "runner_initial_grace_period": "181", + "runner_poll_interval": "11", "ssh_pubkey": "", - "token": "test-token-xyz", "userdata": "", } user_data = aws._build_user_data(**params) - - # Verify all substitutions happened (no template variables remain) - template_vars = [ f'${k}' for k in params ] - for var in template_vars: - assert var not in user_data, f"Template variable {var} was not substituted" - - # Use snapshot to verify the entire output assert user_data == snapshot @@ -100,16 +88,12 @@ def test_build_user_data_missing_params(aws): @pytest.fixture(scope="function") -def complete_params(): - params = { +def complete_params(base_aws_params): + """Extended parameters including AWS-specific configurations""" + return base_aws_params | { "gh_runner_tokens": ["test"], - "home_dir": "/home/ec2-user", "iam_instance_profile": "test", - "image_id": "ami-0772db4c976d21e9b", - "instance_type": "t2.micro", "labels": "", - "region_name": "us-east-1", - "repo": "omsf-eco-infra/awsinfratesting", "root_device_size": 100, "runner_release": "test.tar.gz", "security_group_id": "test", @@ -119,59 +103,75 @@ def complete_params(): {"Key": "Owner", "Value": "test"}, ], } - yield params -@patch.dict('os.environ', { - 'GITHUB_REPOSITORY': 'Open-Athena/ec2-gha', - 'GITHUB_WORKFLOW': 'CI', - 'GITHUB_SERVER_URL': 'https://github.com', - 'GITHUB_RUN_ID': '16725250800' -}) -def test_build_aws_params(complete_params): - user_data_params = { - "cloudwatch_logs_group": "", - "github_run_id": "16725250800", - "github_run_number": "1", - "github_workflow": "CI", - "homedir": "/home/ec2-user", - "labels": "label", - "max_instance_lifetime": "360", - "repo": "omsf-eco-infra/awsinfratesting", - "runner_grace_period": "61", - "runner_initial_grace_period": "181", - "runner_poll_interval": "11", - "runner_release": "test.tar.gz", - "script": "echo 'Hello, World!'", - "ssh_pubkey": "", - "token": "test", - "userdata": "", +@pytest.fixture(scope="function") +def github_env(): + """Common GitHub environment variables for tests""" + return { + 'GITHUB_REPOSITORY': 'Open-Athena/ec2-gha', + 'GITHUB_WORKFLOW': 'CI', + 'GITHUB_WORKFLOW_REF': 'Open-Athena/ec2-gha/.github/workflows/test.yml@refs/heads/main', + 'GITHUB_RUN_NUMBER': '42', + 'GITHUB_SERVER_URL': 'https://github.com', + 'GITHUB_RUN_ID': '16725250800' } - aws = StartAWS(**complete_params) - params = aws._build_aws_params(user_data_params) - - # Test structure without checking exact UserData content - assert params["ImageId"] == "ami-0772db4c976d21e9b" - assert params["InstanceType"] == "t2.micro" - assert params["MinCount"] == 1 - assert params["MaxCount"] == 1 - assert params["SubnetId"] == "test" - assert params["SecurityGroupIds"] == ["test"] - assert params["IamInstanceProfile"] == {"Name": "test"} - assert params["InstanceInitiatedShutdownBehavior"] == "terminate" - assert "UserData" in params - assert params["TagSpecifications"] == [ - { - "ResourceType": "instance", - "Tags": [ - {"Key": "Name", "Value": "test"}, - {"Key": "Owner", "Value": "test"}, - {"Key": "Repository", "Value": "Open-Athena/ec2-gha"}, - {"Key": "Workflow", "Value": "CI"}, - {"Key": "URL", "Value": "https://github.com/Open-Athena/ec2-gha/actions/runs/16725250800"}, - ], + + +def test_build_aws_params_with_idx(complete_params, aws_params_user_data, github_env, snapshot): + """Test _build_aws_params with idx parameter for multi-instance scenarios""" + with patch.dict('os.environ', github_env): + user_data_params = aws_params_user_data + # Remove existing tags to test auto-generated Name tag + params_without_tags = complete_params.copy() + params_without_tags['tags'] = [] + # Add instance_name template for testing + params_without_tags['instance_name'] = '$repo/$name-$idx#$run_number' + aws = StartAWS(**params_without_tags) + + params = aws._build_aws_params(user_data_params, idx=0) + + # Use snapshot to verify the entire structure including UserData + assert params == snapshot + + +def test_build_aws_params(complete_params, aws_params_user_data, github_env, snapshot): + """Test _build_aws_params without idx parameter""" + # Slightly modified github_env without WORKFLOW_REF + env = github_env.copy() + del env['GITHUB_WORKFLOW_REF'] + + with patch.dict('os.environ', env): + user_data_params = aws_params_user_data | {"github_run_number": "1"} + aws = StartAWS(**complete_params) + params = aws._build_aws_params(user_data_params) + + # Use snapshot to verify the entire structure including UserData + assert params == snapshot + + +def test_auto_home_dir(complete_params): + """Test that home_dir is set to AUTO when not provided""" + params = complete_params.copy() + params['home_dir'] = "" + aws = StartAWS(**params) + aws.gh_runner_tokens = ["test-token"] + aws.runner_release = "https://example.com/runner.tar.gz" + + with patch("boto3.client") as mock_client: + mock_ec2 = Mock() + mock_client.return_value = mock_ec2 + + # Mock the run_instances response + mock_ec2.run_instances.return_value = { + "Instances": [{"InstanceId": "i-123456"}] } - ] + + result = aws.create_instances() + + # Verify home_dir was set to AUTO + assert aws.home_dir == AUTO + assert "i-123456" in result def test_modify_root_disk_size(complete_params): @@ -316,12 +316,30 @@ def test_create_instances_missing_release(aws): aws.create_instances() -def test_create_instances_missing_home_dir(aws): - aws.home_dir = "" - with pytest.raises( - ValueError, match="No home directory provided, cannot create instances." - ): - aws.create_instances() +def test_create_instances_sets_auto_home_dir(base_aws_params): + """Test that home_dir is set to AUTO when not provided""" + params = base_aws_params.copy() + params['home_dir'] = "" + + with mock_aws(): + aws = StartAWS(**params) + aws.gh_runner_tokens = ["test-token"] + aws.runner_release = "https://example.com/runner.tar.gz" + + with patch("boto3.client") as mock_client: + mock_ec2 = Mock() + mock_client.return_value = mock_ec2 + + # Mock the run_instances response + mock_ec2.run_instances.return_value = { + "Instances": [{"InstanceId": "i-123456"}] + } + + result = aws.create_instances() + + # Verify home_dir was set to AUTO for runtime detection + assert aws.home_dir == AUTO + assert "i-123456" in result def test_create_instances_missing_tokens(aws): From ed964f518043db65898442b7cc563ad826838d7b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 01:18:14 -0400 Subject: [PATCH 08/69] "merge" v2 --- .github/workflows/demo-gpu-job-seq.yml | 2 +- .github/workflows/demo-gpu-minimal.yml | 2 +- src/ec2_gha/start.py | 37 +- src/ec2_gha/templates/user-script.sh.templ | 361 +++-- tests/__snapshots__/test_start.ambr | 1524 ++++++++------------ 5 files changed, 816 insertions(+), 1110 deletions(-) diff --git a/.github/workflows/demo-gpu-job-seq.yml b/.github/workflows/demo-gpu-job-seq.yml index c66a68a..301df5a 100644 --- a/.github/workflows/demo-gpu-job-seq.yml +++ b/.github/workflows/demo-gpu-job-seq.yml @@ -1,4 +1,4 @@ -name: Demo – comprehensive GPU workload with sequential jobs +name: Demo – simulated GPU workload with sequential jobs on: workflow_dispatch: inputs: diff --git a/.github/workflows/demo-gpu-minimal.yml b/.github/workflows/demo-gpu-minimal.yml index 30bf1b8..1bcb877 100644 --- a/.github/workflows/demo-gpu-minimal.yml +++ b/.github/workflows/demo-gpu-minimal.yml @@ -1,4 +1,4 @@ -name: Demo – minimal EC2 GPU runner +name: Demo – minimal EC2 GPU job on: workflow_dispatch: workflow_call: # Tested by `demos.yml` diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index d5bee80..7622e8f 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -229,6 +229,22 @@ def _build_user_data(self, **kwargs) -> str: try: parsed = Template(template_content) runner_script = parsed.substitute(**kwargs) + + # Strip comment lines to save space (but keep shebang lines) + lines = runner_script.split('\n') + filtered_lines = [] + for line in lines: + stripped = line.strip() + # Keep shebang, empty lines, and non-comment lines + if not stripped or stripped.startswith('#!') or not stripped.startswith('#'): + filtered_lines.append(line) + + runner_script = '\n'.join(filtered_lines) + + # Log the final size + script_size = len(runner_script) + print(f"UserData size: {script_size} bytes ({script_size/16384*100:.1f}% of 16KB limit)") + return runner_script except Exception as e: raise Exception("Error parsing user data template") from e @@ -324,7 +340,26 @@ def create_instances(self) -> dict[str, str]: params = self._build_aws_params(user_data_params, idx=idx) if self.root_device_size > 0: params = self._modify_root_disk_size(ec2, params) - result = ec2.run_instances(**params) + + # Check UserData size before calling AWS + user_data_size = len(params.get("UserData", "")) + if user_data_size > 16384: + raise ValueError( + f"UserData exceeds AWS limit: {user_data_size} bytes (limit: 16384 bytes, " + f"over by: {user_data_size - 16384} bytes). " + f"Template needs to be reduced by at least {user_data_size - 16384} bytes." + ) + + try: + result = ec2.run_instances(**params) + except Exception as e: + if "User data is limited to 16384 bytes" in str(e): + # This shouldn't happen if our check above works, but just in case + raise ValueError( + f"UserData exceeds AWS limit: {user_data_size} bytes (limit: 16384 bytes, " + f"over by: {user_data_size - 16384} bytes)" + ) from e + raise instances = result["Instances"] id = instances[0]["InstanceId"] id_dict[id] = label diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 51e5eda..1406d9a 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -1,17 +1,16 @@ #!/bin/bash set -e -# Helper functions log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $$1" >&2; } # Function to flush CloudWatch logs before shutdown flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } # Create common functions file @@ -19,24 +18,24 @@ cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' #!/bin/bash log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } EOCF # Get metadata (IMDSv2 compatible) get_metadata() { - local path="$$1" - local token=$$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) - if [ -n "$$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $$token" "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" - fi + local path="$$1" + local token=$$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $$token" "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" + fi } logger "EC2-GHA: Starting userdata script" @@ -45,18 +44,18 @@ trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR INSTANCE_ID=$$(get_metadata "instance-id") terminate_instance() { - local reason="$$1" - log "FATAL: $$reason" - log "Terminating instance $$INSTANCE_ID due to setup failure" - - # Try to remove runner if it was partially configured - if [ -f "$$homedir/config.sh" ] && [ -n "$${RUNNER_TOKEN:-}" ]; then - cd "$$homedir" && ./config.sh remove --token "$${RUNNER_TOKEN}" || true - fi - - flush_cloudwatch_logs - shutdown -h now - exit 1 + local reason="$$1" + log "FATAL: $$reason" + log "Terminating instance $$INSTANCE_ID due to setup failure" + + # Try to remove runner if it was partially configured + if [ -f "$$homedir/config.sh" ] && [ -n "$${RUNNER_TOKEN:-}" ]; then + cd "$$homedir" && ./config.sh remove --token "$${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + shutdown -h now + exit 1 } # Trap errors and ensure termination on failure @@ -66,19 +65,19 @@ trap 'terminate_instance "Setup script failed with error on line $$LINENO"' ERR REGISTRATION_TIMEOUT="$runner_registration_timeout" # Validate timeout is a number, default to 300 if not if ! [[ "$$REGISTRATION_TIMEOUT" =~ ^[0-9]+$$ ]]; then - logger "EC2-GHA: Invalid timeout '$$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 + logger "EC2-GHA: Invalid timeout '$$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" ( - log "Watchdog: Starting $$REGISTRATION_TIMEOUT second timeout" - sleep $$REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then - log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $$REGISTRATION_TIMEOUT seconds" - else - log "Watchdog: Registration marker found, exiting normally" - fi + log "Watchdog: Starting $$REGISTRATION_TIMEOUT second timeout" + sleep $$REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $$REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi ) & REGISTRATION_WATCHDOG_PID=$$! log "Watchdog PID: $$REGISTRATION_WATCHDOG_PID" @@ -92,37 +91,37 @@ homedir="$homedir" # Determine home directory if not specified or set to AUTO if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - # Check for common cloud-init created users - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$$user" &>/dev/null; then - homedir="/home/$$user" - log "Auto-detected: $$homedir" - break - fi - done - - # Fallback if no standard user found - if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then - # Get the first non-root user's home directory that actually exists - homedir=$$(getent passwd | awk -F: '$$3 >= 1000 && $$3 < 65534 && $$6 ~ /^\/home\// {print $$6}' | while read dir; do - if [ -d "$$dir" ]; then - echo "$$dir" - break - fi - done) - - if [ -z "$$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $$homedir" - else - # Use stat to get the actual owner of the directory - owner=$$(stat -c "%U" "$$homedir" 2>/dev/null || stat -f "%Su" "$$homedir" 2>/dev/null) - log "Detected: $$homedir ($$owner)" - fi + # Try to find the default non-root user's home directory + # Check for common cloud-init created users + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$$user" &>/dev/null; then + homedir="/home/$$user" + log "Auto-detected: $$homedir" + break fi + done + + # Fallback if no standard user found + if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then + # Get the first non-root user's home directory that actually exists + homedir=$$(getent passwd | awk -F: '$$3 >= 1000 && $$3 < 65534 && $$6 ~ /^\/home\// {print $$6}' | while read dir; do + if [ -d "$$dir" ]; then + echo "$$dir" + break + fi + done) + + if [ -z "$$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $$homedir" + else + # Use stat to get the actual owner of the directory + owner=$$(stat -c "%U" "$$homedir" 2>/dev/null || stat -f "%Su" "$$homedir" 2>/dev/null) + log "Detected: $$homedir ($$owner)" + fi + fi else - log "Using: $$homedir" + log "Using: $$homedir" fi # Fetch instance metadata before redirecting output @@ -144,21 +143,21 @@ nohup bash -c "sleep $${MAX_LIFETIME_MINUTES}m && echo '[$$(date)] Maximum lifet # Configure CloudWatch Logs if enabled if [ "$cloudwatch_logs_group" != "" ]; then - log "Installing CloudWatch agent" - # Use a subshell to prevent CloudWatch failures from stopping the entire script - ( + log "Installing CloudWatch agent" + # Use a subshell to prevent CloudWatch failures from stopping the entire script + ( # Wait for dpkg lock to be released (up to 2 minutes) log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $$timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($$timeout seconds remaining)" - sleep 5 - timeout=$$((timeout - 5)) + if [ $$timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($$timeout seconds remaining)" + sleep 5 + timeout=$$((timeout - 5)) done # Download and install CloudWatch agent @@ -167,7 +166,7 @@ if [ "$cloudwatch_logs_group" != "" ]; then rm amazon-cloudwatch-agent.deb # Configure CloudWatch agent - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { "agent": { "run_as_user": "cwagent" @@ -176,42 +175,12 @@ if [ "$cloudwatch_logs_group" != "" ]; then "logs_collected": { "files": { "collect_list": [ - { - "file_path": "/var/log/runner-setup.log", - "log_group_name": "$cloudwatch_logs_group", - "log_stream_name": "{instance_id}/runner-setup", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-started-hook.log", - "log_group_name": "$cloudwatch_logs_group", - "log_stream_name": "{instance_id}/job-started", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-completed-hook.log", - "log_group_name": "$cloudwatch_logs_group", - "log_stream_name": "{instance_id}/job-completed", - "timezone": "UTC" - }, - { - "file_path": "/tmp/termination-check.log", - "log_group_name": "$cloudwatch_logs_group", - "log_stream_name": "{instance_id}/termination", - "timezone": "UTC" - }, - { - "file_path": "$$homedir/_diag/Runner_**.log", - "log_group_name": "$cloudwatch_logs_group", - "log_stream_name": "{instance_id}/runner-diag", - "timezone": "UTC" - }, - { - "file_path": "$$homedir/_diag/Worker_**.log", - "log_group_name": "$cloudwatch_logs_group", - "log_stream_name": "{instance_id}/worker-diag", - "timezone": "UTC" - } + { "file_path": "/var/log/runner-setup.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/tmp/job-started-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, + { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, + { "file_path": "/tmp/termination-check.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, + { "file_path": "$homedir/_diag/Runner_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, + { "file_path": "$homedir/_diag/Worker_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } ] } } @@ -221,36 +190,36 @@ EOF # Start CloudWatch agent /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi # Configure SSH access if public key provided if [ -n "$ssh_pubkey" ]; then - log "Configuring SSH access" + log "Configuring SSH access" - # Determine the default user based on the home directory owner - DEFAULT_USER=$$(stat -c "%U" "$$homedir" 2>/dev/null || echo "root") + # Determine the default user based on the home directory owner + DEFAULT_USER=$$(stat -c "%U" "$$homedir" 2>/dev/null || echo "root") - # Create .ssh directory if it doesn't exist - mkdir -p "$homedir/.ssh" - chmod 700 "$homedir/.ssh" + # Create .ssh directory if it doesn't exist + mkdir -p "$homedir/.ssh" + chmod 700 "$homedir/.ssh" - # Add the public key to authorized_keys - echo "$ssh_pubkey" >> "$$homedir/.ssh/authorized_keys" - chmod 600 "$$homedir/.ssh/authorized_keys" + # Add the public key to authorized_keys + echo "$ssh_pubkey" >> "$$homedir/.ssh/authorized_keys" + chmod 600 "$$homedir/.ssh/authorized_keys" - # Set proper ownership - if [ "$$DEFAULT_USER" != "root" ]; then - chown -R "$$DEFAULT_USER:$$DEFAULT_USER" "$$homedir/.ssh" - fi + # Set proper ownership + if [ "$$DEFAULT_USER" != "root" ]; then + chown -R "$$DEFAULT_USER:$$DEFAULT_USER" "$$homedir/.ssh" + fi - log "SSH key added for user $$DEFAULT_USER" + log "SSH key added for user $$DEFAULT_USER" fi log "Working directory: $$homedir" @@ -263,12 +232,12 @@ export RUNNER_ALLOW_RUNASROOT=1 # Detect architecture and download appropriate runner ARCH=$$(uname -m) if [ "$$ARCH" = "aarch64" ] || [ "$$ARCH" = "arm64" ]; then - # For ARM, replace x64 with arm64 in the URL - RUNNER_URL=$$(echo "$runner_release" | sed 's/x64/arm64/g') - log "ARM detected, using: $$RUNNER_URL" + # For ARM, replace x64 with arm64 in the URL + RUNNER_URL=$$(echo "$runner_release" | sed 's/x64/arm64/g') + log "ARM detected, using: $$RUNNER_URL" else - RUNNER_URL="$runner_release" - log "x64 detected, using: $$RUNNER_URL" + RUNNER_URL="$runner_release" + log "x64 detected, using: $$RUNNER_URL" fi curl -L $$RUNNER_URL -o runner.tar.gz log "Extracting runner" @@ -278,52 +247,56 @@ tar --no-overwrite-dir -xzf runner.tar.gz cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 +V="/var/run/github-runner" echo "[$$(date)] ${log_prefix_job_started} $${GITHUB_JOB}" -mkdir -p /var/run/github-runner-jobs -echo '{"status":"running"}' > /var/run/github-runner-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job -touch /var/run/github-runner-last-activity -# Mark that at least one job has run -touch /var/run/github-runner-has-run-job +mkdir -p $$V-jobs +echo '{"status":"running"}' > $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job +touch $$V-last-activity $$V-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 +V="/var/run/github-runner" echo "[$$(date)] ${log_prefix_job_completed} $${GITHUB_JOB}" -rm -f /var/run/github-runner-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job -touch /var/run/github-runner-last-activity +rm -f $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job +touch $$V-last-activity EOFC -cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' +cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh -A="/var/run/github-runner-last-activity" -J="/var/run/github-runner-jobs" -H="/var/run/github-runner-has-run-job" -[ ! -f "$$A" ] && touch "$$A" -L=$$(stat -c %Y "$$A" 2>/dev/null || echo 0) -I=$$(($$(date +%s)-L)) -# Use initial grace period only if no job has ever run -if [ -f "$$H" ]; then G=$${RUNNER_GRACE_PERIOD:-60}; else G=$${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi -R=$$(grep -l '"status":"running"' $$J/*.job 2>/dev/null | wc -l || echo 0) -if [ $$R -eq 0 ] && [ $$I -gt $$G ]; then - echo "[$$(date)] Terminating: idle $$I > grace $$G, proceeding with termination" - [ -f "$$homedir/config.sh" ] && cd "$$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true +V="/var/run/github-runner" +A="\$$V-last-activity" +J="\$$V-jobs" +H="\$$V-has-run-job" +D="$$homedir" +T="$token" + +[ ! -f "\$$A" ] && touch "\$$A" +L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) +N=\$$(date +%s) +I=\$$((N-L)) +[ -f "\$$H" ] && G=\$${RUNNER_GRACE_PERIOD:-60} || G=\$${RUNNER_INITIAL_GRACE_PERIOD:-180} +R=\$$(grep -l '"status":"running"' \$$J/*.job 2>/dev/null | wc -l || echo 0) + +if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then + log "TERMINATING: idle \$$I > grace \$$G" + if [ -f "\$$D/config.sh" ]; then + cd "\$$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true sleep 2 - if [ -f "$$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $token; then - echo "[$$(date)] Runner removed from GitHub successfully" - fi - - flush_cloudwatch_logs - sudo shutdown -h now "Runner terminating after idle timeout" + log "Deregistering runner..." + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$$T 2>&1 + log "Deregistration exit: \$$?" + else + log "ERROR: config.sh not found at \$$D" + fi + flush_cloudwatch_logs + log "Shutting down" + sudo shutdown -h now else - if [ $$R -gt 0 ]; then - echo "[$$(date)] $$R job(s) still running, not terminating" - else - echo "[$$(date)] No jobs running, idle $$I seconds (grace period: $$G seconds)" - fi + [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" fi EOFT @@ -338,10 +311,11 @@ echo "RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" >> .env echo "RUNNER_POLL_INTERVAL=$runner_poll_interval" >> .env # Set up job tracking directory -mkdir -p /var/run/github-runner-jobs +V="/var/run/github-runner" +mkdir -p $$V-jobs # Create initial activity timestamp -touch /var/run/github-runner-last-activity +touch $$V-last-activity # Set up periodic termination check using systemd cat > /etc/systemd/system/runner-termination-check.service << 'EOF' @@ -387,15 +361,15 @@ METADATA_LABELS="$${METADATA_LABELS},instance-type:$${INSTANCE_TYPE}" # Add GitHub workflow metadata passed from the action if [ -n "$github_workflow" ]; then - # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$$(echo "$github_workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="$${METADATA_LABELS},workflow:$${WORKFLOW_LABEL}" + # Replace spaces and special chars in workflow name for label compatibility + WORKFLOW_LABEL=$$(echo "$github_workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="$${METADATA_LABELS},workflow:$${WORKFLOW_LABEL}" fi if [ -n "$github_run_id" ]; then - METADATA_LABELS="$${METADATA_LABELS},run-id:$github_run_id" + METADATA_LABELS="$${METADATA_LABELS},run-id:$github_run_id" fi if [ -n "$github_run_number" ]; then - METADATA_LABELS="$${METADATA_LABELS},run-number:$github_run_number" + METADATA_LABELS="$${METADATA_LABELS},run-number:$github_run_number" fi # Combine provided labels (user + runner-xxx) with instance metadata labels @@ -411,10 +385,10 @@ export RUNNER_TOKEN="$token" # Attempt to register. If a runner with the same name exists, config.sh will fail # We use --unattended to prevent interactive prompts if ! ./config.sh --url https://github.com/$repo --token $token --labels "$${ALL_LABELS}" --name "$${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '$${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log_error "Failed to register runner '$${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" fi log "Runner registered successfully" @@ -424,19 +398,20 @@ touch /var/run/github-runner-registered ls -la /var/run/github-runner-registered # Read watchdog PID from file (survives exec redirect) if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $$WATCHDOG_PID 2>/dev/null; then - kill $$WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $$WATCHDOG_PID)" - else - log "Watchdog process $$WATCHDOG_PID already terminated" - fi - rm -f /var/run/github-runner-watchdog.pid + WATCHDOG_PID=$$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $$WATCHDOG_PID 2>/dev/null; then + kill $$WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $$WATCHDOG_PID)" + else + log "Watchdog process $$WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid fi log "Starting runner" # Create marker file for cleanup service touch /var/run/github-runner-started + # Ensure CloudWatch agent can read diagnostic logs # The cwagent user needs to traverse into $homedir to reach _diag # Make $homedir world-executable (but not readable) so cwagent can traverse it diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index 93a9bdf..dc3becd 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -43,43 +43,39 @@ 'UserData': ''' #!/bin/bash set -e - # Helper functions log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - # Function to flush CloudWatch logs before shutdown flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } - # Create common functions file cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' #!/bin/bash log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } EOCF - # Get metadata (IMDSv2 compatible) get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - fi + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi } logger "EC2-GHA: Starting userdata script" @@ -88,129 +84,109 @@ INSTANCE_ID=$(get_metadata "instance-id") terminate_instance() { - local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" - # Try to remove runner if it was partially configured - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi - flush_cloudwatch_logs - shutdown -h now - exit 1 + flush_cloudwatch_logs + shutdown -h now + exit 1 } - # Trap errors and ensure termination on failure trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - # Set up registration timeout failsafe REGISTRATION_TIMEOUT="300" - # Validate timeout is a number, default to 300 if not if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then - log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" - else - log "Watchdog: Registration marker found, exiting normally" - fi + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - # Save watchdog PID to file so it survives exec redirect echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid - # Initialize homedir from template variable homedir="/home/ec2-user" - # Determine home directory if not specified or set to AUTO if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - # Check for common cloud-init created users - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) - # Fallback if no standard user found - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Get the first non-root user's home directory that actually exists - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - # Use stat to get the actual owner of the directory - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" fi + fi else - log "Using: $homedir" + log "Using: $homedir" fi - # Fetch instance metadata before redirecting output INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - # Redirect runner setup logs to a file for CloudWatch exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & - # Configure CloudWatch Logs if enabled if [ "" != "" ]; then - log "Installing CloudWatch agent" - # Use a subshell to prevent CloudWatch failures from stopping the entire script - ( + log "Installing CloudWatch agent" + ( - # Wait for dpkg lock to be released (up to 2 minutes) log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) done - # Download and install CloudWatch agent wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb - # Configure CloudWatch agent - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { "agent": { "run_as_user": "cwagent" @@ -219,42 +195,12 @@ "logs_collected": { "files": { "collect_list": [ - { - "file_path": "/var/log/runner-setup.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/runner-setup", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-started-hook.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/job-started", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-completed-hook.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/job-completed", - "timezone": "UTC" - }, - { - "file_path": "/tmp/termination-check.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/termination", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Runner_**.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/runner-diag", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Worker_**.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/worker-diag", - "timezone": "UTC" - } + { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, + { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, + { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } ] } } @@ -262,38 +208,32 @@ } EOF - # Start CloudWatch agent /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi - # Configure SSH access if public key provided if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - log "Configuring SSH access" + log "Configuring SSH access" - # Determine the default user based on the home directory owner - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - # Create .ssh directory if it doesn't exist - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" - # Add the public key to authorized_keys - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" - # Set proper ownership - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi - log "SSH key added for user $DEFAULT_USER" + log "SSH key added for user $DEFAULT_USER" fi log "Working directory: $homedir" @@ -303,68 +243,75 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 - # Detect architecture and download appropriate runner ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - # For ARM, replace x64 with arm64 in the URL - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" fi curl -L $RUNNER_URL -o runner.tar.gz log "Extracting runner" - # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` tar --no-overwrite-dir -xzf runner.tar.gz - # Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job started: ${GITHUB_JOB}" - mkdir -p /var/run/github-runner-jobs - echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + mkdir -p $V-jobs + echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity $V-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' + cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh - A="/var/run/github-runner-last-activity" - J="/var/run/github-runner-jobs" - [ ! -f "$A" ] && touch "$A" - L=$(stat -c %Y "$A" 2>/dev/null || echo 0) - I=$(($(date +%s)-L)) - if ls $J/*.job 2>/dev/null | grep -q .; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi - R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) - if [ $R -eq 0 ] && [ $I -gt $G ]; then - echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" - [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + V="/var/run/github-runner" + A="\$V-last-activity" + J="\$V-jobs" + H="\$V-has-run-job" + D="$homedir" + T="test" + + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" + if [ -f "\$D/config.sh" ]; then + cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true sleep 2 - if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then - echo "[$(date)] Runner removed from GitHub successfully" - fi - - flush_cloudwatch_logs - sudo shutdown -h now "Runner terminating after idle timeout" + log "Deregistering runner..." + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 + log "Deregistration exit: \$?" + else + log "ERROR: config.sh not found at \$D" + fi + flush_cloudwatch_logs + log "Shutting down" + sudo shutdown -h now else - echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=$homedir" >> .env @@ -372,13 +319,11 @@ echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env echo "RUNNER_POLL_INTERVAL=11" >> .env - # Set up job tracking directory - mkdir -p /var/run/github-runner-jobs + V="/var/run/github-runner" + mkdir -p $V-jobs - # Create initial activity timestamp - touch /var/run/github-runner-last-activity + touch $V-last-activity - # Set up periodic termination check using systemd cat > /etc/systemd/system/runner-termination-check.service << 'EOF' [Unit] Description=Check GitHub runner termination conditions @@ -402,84 +347,65 @@ WantedBy=timers.target EOF - # Enable and start the timer systemctl daemon-reload systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - # Get instance metadata for descriptive runner name INSTANCE_ID=$(get_metadata "instance-id") INSTANCE_TYPE=$(get_metadata "instance-type") - # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-${INSTANCE_ID}" - # Build additional labels with metadata for easier correlation - # These will be visible in the GitHub runner management UI METADATA_LABELS="" METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - # Add GitHub workflow metadata passed from the action if [ -n "CI" ]; then - # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" fi if [ -n "1" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:1" + METADATA_LABELS="${METADATA_LABELS},run-number:1" fi - # Combine provided labels (user + runner-xxx) with instance metadata labels - # The label variable already contains user labels and the critical runner-xxx label from Python ALL_LABELS="label${METADATA_LABELS}" log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" log "Runner name: ${RUNNER_NAME}" log "Labels: ${ALL_LABELS}" - # Export token for error handler export RUNNER_TOKEN="test" - # Attempt to register. If a runner with the same name exists, config.sh will fail - # We use --unattended to prevent interactive prompts if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" fi log "Runner registered successfully" - # Mark registration as complete and kill the watchdog log "Creating registration marker" touch /var/run/github-runner-registered ls -la /var/run/github-runner-registered - # Read watchdog PID from file (survives exec redirect) if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi - rm -f /var/run/github-runner-watchdog.pid + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid fi log "Starting runner" - # Create marker file for cleanup service touch /var/run/github-runner-started - # Ensure CloudWatch agent can read diagnostic logs - # The cwagent user needs to traverse into /home/ec2-user to reach _diag - # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir log "Made $homedir traversable for CloudWatch agent" - # Create _diag directory if it doesn't exist mkdir -p $homedir/_diag - # The _diag files are already world-readable by default, just ensure the directory is too chmod 755 $homedir/_diag ./run.sh @@ -526,43 +452,39 @@ 'UserData': ''' #!/bin/bash set -e - # Helper functions log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - # Function to flush CloudWatch logs before shutdown flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } - # Create common functions file cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' #!/bin/bash log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } EOCF - # Get metadata (IMDSv2 compatible) get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - fi + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi } logger "EC2-GHA: Starting userdata script" @@ -571,129 +493,109 @@ INSTANCE_ID=$(get_metadata "instance-id") terminate_instance() { - local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" - # Try to remove runner if it was partially configured - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi - flush_cloudwatch_logs - shutdown -h now - exit 1 + flush_cloudwatch_logs + shutdown -h now + exit 1 } - # Trap errors and ensure termination on failure trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - # Set up registration timeout failsafe REGISTRATION_TIMEOUT="300" - # Validate timeout is a number, default to 300 if not if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then - log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" - else - log "Watchdog: Registration marker found, exiting normally" - fi + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - # Save watchdog PID to file so it survives exec redirect echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid - # Initialize homedir from template variable homedir="/home/ec2-user" - # Determine home directory if not specified or set to AUTO if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - # Check for common cloud-init created users - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) - # Fallback if no standard user found - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Get the first non-root user's home directory that actually exists - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - # Use stat to get the actual owner of the directory - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" fi + fi else - log "Using: $homedir" + log "Using: $homedir" fi - # Fetch instance metadata before redirecting output INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - # Redirect runner setup logs to a file for CloudWatch exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & - # Configure CloudWatch Logs if enabled if [ "" != "" ]; then - log "Installing CloudWatch agent" - # Use a subshell to prevent CloudWatch failures from stopping the entire script - ( + log "Installing CloudWatch agent" + ( - # Wait for dpkg lock to be released (up to 2 minutes) log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) done - # Download and install CloudWatch agent wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb - # Configure CloudWatch agent - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { "agent": { "run_as_user": "cwagent" @@ -702,42 +604,12 @@ "logs_collected": { "files": { "collect_list": [ - { - "file_path": "/var/log/runner-setup.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/runner-setup", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-started-hook.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/job-started", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-completed-hook.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/job-completed", - "timezone": "UTC" - }, - { - "file_path": "/tmp/termination-check.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/termination", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Runner_**.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/runner-diag", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Worker_**.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/worker-diag", - "timezone": "UTC" - } + { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, + { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, + { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } ] } } @@ -745,38 +617,32 @@ } EOF - # Start CloudWatch agent /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi - # Configure SSH access if public key provided if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - log "Configuring SSH access" + log "Configuring SSH access" - # Determine the default user based on the home directory owner - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - # Create .ssh directory if it doesn't exist - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" - # Add the public key to authorized_keys - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" - # Set proper ownership - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi - log "SSH key added for user $DEFAULT_USER" + log "SSH key added for user $DEFAULT_USER" fi log "Working directory: $homedir" @@ -786,68 +652,75 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 - # Detect architecture and download appropriate runner ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - # For ARM, replace x64 with arm64 in the URL - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" fi curl -L $RUNNER_URL -o runner.tar.gz log "Extracting runner" - # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` tar --no-overwrite-dir -xzf runner.tar.gz - # Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job started: ${GITHUB_JOB}" - mkdir -p /var/run/github-runner-jobs - echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + mkdir -p $V-jobs + echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity $V-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' + cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh - A="/var/run/github-runner-last-activity" - J="/var/run/github-runner-jobs" - [ ! -f "$A" ] && touch "$A" - L=$(stat -c %Y "$A" 2>/dev/null || echo 0) - I=$(($(date +%s)-L)) - if ls $J/*.job 2>/dev/null | grep -q .; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi - R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) - if [ $R -eq 0 ] && [ $I -gt $G ]; then - echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" - [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + V="/var/run/github-runner" + A="\$V-last-activity" + J="\$V-jobs" + H="\$V-has-run-job" + D="$homedir" + T="test" + + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" + if [ -f "\$D/config.sh" ]; then + cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true sleep 2 - if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then - echo "[$(date)] Runner removed from GitHub successfully" - fi - - flush_cloudwatch_logs - sudo shutdown -h now "Runner terminating after idle timeout" + log "Deregistering runner..." + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 + log "Deregistration exit: \$?" + else + log "ERROR: config.sh not found at \$D" + fi + flush_cloudwatch_logs + log "Shutting down" + sudo shutdown -h now else - echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=$homedir" >> .env @@ -855,13 +728,11 @@ echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env echo "RUNNER_POLL_INTERVAL=11" >> .env - # Set up job tracking directory - mkdir -p /var/run/github-runner-jobs + V="/var/run/github-runner" + mkdir -p $V-jobs - # Create initial activity timestamp - touch /var/run/github-runner-last-activity + touch $V-last-activity - # Set up periodic termination check using systemd cat > /etc/systemd/system/runner-termination-check.service << 'EOF' [Unit] Description=Check GitHub runner termination conditions @@ -885,84 +756,65 @@ WantedBy=timers.target EOF - # Enable and start the timer systemctl daemon-reload systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - # Get instance metadata for descriptive runner name INSTANCE_ID=$(get_metadata "instance-id") INSTANCE_TYPE=$(get_metadata "instance-type") - # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-${INSTANCE_ID}" - # Build additional labels with metadata for easier correlation - # These will be visible in the GitHub runner management UI METADATA_LABELS="" METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - # Add GitHub workflow metadata passed from the action if [ -n "CI" ]; then - # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" fi if [ -n "42" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:42" + METADATA_LABELS="${METADATA_LABELS},run-number:42" fi - # Combine provided labels (user + runner-xxx) with instance metadata labels - # The label variable already contains user labels and the critical runner-xxx label from Python ALL_LABELS="label${METADATA_LABELS}" log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" log "Runner name: ${RUNNER_NAME}" log "Labels: ${ALL_LABELS}" - # Export token for error handler export RUNNER_TOKEN="test" - # Attempt to register. If a runner with the same name exists, config.sh will fail - # We use --unattended to prevent interactive prompts if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" fi log "Runner registered successfully" - # Mark registration as complete and kill the watchdog log "Creating registration marker" touch /var/run/github-runner-registered ls -la /var/run/github-runner-registered - # Read watchdog PID from file (survives exec redirect) if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi - rm -f /var/run/github-runner-watchdog.pid + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid fi log "Starting runner" - # Create marker file for cleanup service touch /var/run/github-runner-started - # Ensure CloudWatch agent can read diagnostic logs - # The cwagent user needs to traverse into /home/ec2-user to reach _diag - # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir log "Made $homedir traversable for CloudWatch agent" - # Create _diag directory if it doesn't exist mkdir -p $homedir/_diag - # The _diag files are already world-readable by default, just ensure the directory is too chmod 755 $homedir/_diag ./run.sh @@ -973,43 +825,39 @@ ''' #!/bin/bash set -e - # Helper functions log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - # Function to flush CloudWatch logs before shutdown flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } - # Create common functions file cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' #!/bin/bash log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } EOCF - # Get metadata (IMDSv2 compatible) get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - fi + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi } logger "EC2-GHA: Starting userdata script" @@ -1018,129 +866,109 @@ INSTANCE_ID=$(get_metadata "instance-id") terminate_instance() { - local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" - # Try to remove runner if it was partially configured - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi - flush_cloudwatch_logs - shutdown -h now - exit 1 + flush_cloudwatch_logs + shutdown -h now + exit 1 } - # Trap errors and ensure termination on failure trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - # Set up registration timeout failsafe REGISTRATION_TIMEOUT="300" - # Validate timeout is a number, default to 300 if not if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then - log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" - else - log "Watchdog: Registration marker found, exiting normally" - fi + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - # Save watchdog PID to file so it survives exec redirect echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid - # Initialize homedir from template variable homedir="/home/ec2-user" - # Determine home directory if not specified or set to AUTO if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - # Check for common cloud-init created users - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done - - # Fallback if no standard user found - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Get the first non-root user's home directory that actually exists - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - # Use stat to get the actual owner of the directory - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" fi + fi else - log "Using: $homedir" + log "Using: $homedir" fi - # Fetch instance metadata before redirecting output INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - # Redirect runner setup logs to a file for CloudWatch exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & - # Configure CloudWatch Logs if enabled if [ "" != "" ]; then - log "Installing CloudWatch agent" - # Use a subshell to prevent CloudWatch failures from stopping the entire script - ( + log "Installing CloudWatch agent" + ( - # Wait for dpkg lock to be released (up to 2 minutes) log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) done - # Download and install CloudWatch agent wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb - # Configure CloudWatch agent - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { "agent": { "run_as_user": "cwagent" @@ -1149,42 +977,12 @@ "logs_collected": { "files": { "collect_list": [ - { - "file_path": "/var/log/runner-setup.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/runner-setup", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-started-hook.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/job-started", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-completed-hook.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/job-completed", - "timezone": "UTC" - }, - { - "file_path": "/tmp/termination-check.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/termination", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Runner_**.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/runner-diag", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Worker_**.log", - "log_group_name": "", - "log_stream_name": "{instance_id}/worker-diag", - "timezone": "UTC" - } + { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, + { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, + { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } ] } } @@ -1192,38 +990,32 @@ } EOF - # Start CloudWatch agent /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi - # Configure SSH access if public key provided if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - log "Configuring SSH access" + log "Configuring SSH access" - # Determine the default user based on the home directory owner - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - # Create .ssh directory if it doesn't exist - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" - # Add the public key to authorized_keys - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" - # Set proper ownership - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi - log "SSH key added for user $DEFAULT_USER" + log "SSH key added for user $DEFAULT_USER" fi log "Working directory: $homedir" @@ -1233,76 +1025,75 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 - # Detect architecture and download appropriate runner ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - # For ARM, replace x64 with arm64 in the URL - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" fi curl -L $RUNNER_URL -o runner.tar.gz log "Extracting runner" - # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` tar --no-overwrite-dir -xzf runner.tar.gz - # Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job started: ${GITHUB_JOB}" - mkdir -p /var/run/github-runner-jobs - echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity - # Mark that at least one job has run - touch /var/run/github-runner-has-run-job + mkdir -p $V-jobs + echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity $V-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' + cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh - A="/var/run/github-runner-last-activity" - J="/var/run/github-runner-jobs" - H="/var/run/github-runner-has-run-job" - [ ! -f "$A" ] && touch "$A" - L=$(stat -c %Y "$A" 2>/dev/null || echo 0) - I=$(($(date +%s)-L)) - # Use initial grace period only if no job has ever run - if [ -f "$H" ]; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi - R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) - if [ $R -eq 0 ] && [ $I -gt $G ]; then - echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" - [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + V="/var/run/github-runner" + A="\$V-last-activity" + J="\$V-jobs" + H="\$V-has-run-job" + D="$homedir" + T="test" + + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" + if [ -f "\$D/config.sh" ]; then + cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true sleep 2 - if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then - echo "[$(date)] Runner removed from GitHub successfully" - fi - - flush_cloudwatch_logs - sudo shutdown -h now "Runner terminating after idle timeout" + log "Deregistering runner..." + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 + log "Deregistration exit: \$?" + else + log "ERROR: config.sh not found at \$D" + fi + flush_cloudwatch_logs + log "Shutting down" + sudo shutdown -h now else - if [ $R -gt 0 ]; then - echo "[$(date)] $R job(s) still running, not terminating" - else - echo "[$(date)] No jobs running, idle $I seconds (grace period: $G seconds)" - fi + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=$homedir" >> .env @@ -1310,13 +1101,11 @@ echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env echo "RUNNER_POLL_INTERVAL=11" >> .env - # Set up job tracking directory - mkdir -p /var/run/github-runner-jobs + V="/var/run/github-runner" + mkdir -p $V-jobs - # Create initial activity timestamp - touch /var/run/github-runner-last-activity + touch $V-last-activity - # Set up periodic termination check using systemd cat > /etc/systemd/system/runner-termination-check.service << 'EOF' [Unit] Description=Check GitHub runner termination conditions @@ -1340,84 +1129,65 @@ WantedBy=timers.target EOF - # Enable and start the timer systemctl daemon-reload systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - # Get instance metadata for descriptive runner name INSTANCE_ID=$(get_metadata "instance-id") INSTANCE_TYPE=$(get_metadata "instance-type") - # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-${INSTANCE_ID}" - # Build additional labels with metadata for easier correlation - # These will be visible in the GitHub runner management UI METADATA_LABELS="" METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - # Add GitHub workflow metadata passed from the action if [ -n "CI" ]; then - # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" fi if [ -n "42" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:42" + METADATA_LABELS="${METADATA_LABELS},run-number:42" fi - # Combine provided labels (user + runner-xxx) with instance metadata labels - # The label variable already contains user labels and the critical runner-xxx label from Python ALL_LABELS="label${METADATA_LABELS}" log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" log "Runner name: ${RUNNER_NAME}" log "Labels: ${ALL_LABELS}" - # Export token for error handler export RUNNER_TOKEN="test" - # Attempt to register. If a runner with the same name exists, config.sh will fail - # We use --unattended to prevent interactive prompts if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" fi log "Runner registered successfully" - # Mark registration as complete and kill the watchdog log "Creating registration marker" touch /var/run/github-runner-registered ls -la /var/run/github-runner-registered - # Read watchdog PID from file (survives exec redirect) if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi - rm -f /var/run/github-runner-watchdog.pid + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid fi log "Starting runner" - # Create marker file for cleanup service touch /var/run/github-runner-started - # Ensure CloudWatch agent can read diagnostic logs - # The cwagent user needs to traverse into /home/ec2-user to reach _diag - # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir log "Made $homedir traversable for CloudWatch agent" - # Create _diag directory if it doesn't exist mkdir -p $homedir/_diag - # The _diag files are already world-readable by default, just ensure the directory is too chmod 755 $homedir/_diag ./run.sh @@ -1427,43 +1197,39 @@ ''' #!/bin/bash set -e - # Helper functions log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - # Function to flush CloudWatch logs before shutdown flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } - # Create common functions file cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' #!/bin/bash log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || \ - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a stop -m ec2 2>/dev/null || true - fi + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null \ + || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ + || true + fi } EOCF - # Get metadata (IMDSv2 compatible) get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" - fi + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi } logger "EC2-GHA: Starting userdata script" @@ -1472,129 +1238,109 @@ INSTANCE_ID=$(get_metadata "instance-id") terminate_instance() { - local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local reason="$1" + log "FATAL: $reason" + log "Terminating instance $INSTANCE_ID due to setup failure" - # Try to remove runner if it was partially configured - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi - flush_cloudwatch_logs - shutdown -h now - exit 1 + flush_cloudwatch_logs + shutdown -h now + exit 1 } - # Trap errors and ensure termination on failure trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - # Set up registration timeout failsafe REGISTRATION_TIMEOUT="300" - # Validate timeout is a number, default to 300 if not if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 + logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" + REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then - log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" - else - log "Watchdog: Registration marker found, exiting normally" - fi + log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" + sleep $REGISTRATION_TIMEOUT + if [ ! -f /var/run/github-runner-registered ]; then + log "Watchdog: Registration marker not found after timeout" + terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + else + log "Watchdog: Registration marker found, exiting normally" + fi ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - # Save watchdog PID to file so it survives exec redirect echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid - # Initialize homedir from template variable homedir="/home/ec2-user" - # Determine home directory if not specified or set to AUTO if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - # Check for common cloud-init created users - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done - - # Fallback if no standard user found - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - # Get the first non-root user's home directory that actually exists - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - # Use stat to get the actual owner of the directory - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + log "Auto-detected: $homedir" + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + log "Using fallback: $homedir" + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + log "Detected: $homedir ($owner)" fi + fi else - log "Using: $homedir" + log "Using: $homedir" fi - # Fetch instance metadata before redirecting output INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - # Redirect runner setup logs to a file for CloudWatch exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - # Set up maximum lifetime timeout - do this early to ensure cleanup MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & - # Configure CloudWatch Logs if enabled if [ "/aws/ec2/github-runners" != "" ]; then - log "Installing CloudWatch agent" - # Use a subshell to prevent CloudWatch failures from stopping the entire script - ( + log "Installing CloudWatch agent" + ( - # Wait for dpkg lock to be released (up to 2 minutes) log "Waiting for dpkg lock to be released..." timeout=120 while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) done - # Download and install CloudWatch agent wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb - # Configure CloudWatch agent - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'EOF' + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { "agent": { "run_as_user": "cwagent" @@ -1603,42 +1349,12 @@ "logs_collected": { "files": { "collect_list": [ - { - "file_path": "/var/log/runner-setup.log", - "log_group_name": "/aws/ec2/github-runners", - "log_stream_name": "{instance_id}/runner-setup", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-started-hook.log", - "log_group_name": "/aws/ec2/github-runners", - "log_stream_name": "{instance_id}/job-started", - "timezone": "UTC" - }, - { - "file_path": "/tmp/job-completed-hook.log", - "log_group_name": "/aws/ec2/github-runners", - "log_stream_name": "{instance_id}/job-completed", - "timezone": "UTC" - }, - { - "file_path": "/tmp/termination-check.log", - "log_group_name": "/aws/ec2/github-runners", - "log_stream_name": "{instance_id}/termination", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Runner_**.log", - "log_group_name": "/aws/ec2/github-runners", - "log_stream_name": "{instance_id}/runner-diag", - "timezone": "UTC" - }, - { - "file_path": "$homedir/_diag/Worker_**.log", - "log_group_name": "/aws/ec2/github-runners", - "log_stream_name": "{instance_id}/worker-diag", - "timezone": "UTC" - } + { "file_path": "/var/log/runner-setup.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/tmp/job-started-hook.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, + { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, + { "file_path": "/tmp/termination-check.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, + { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } ] } } @@ -1646,38 +1362,32 @@ } EOF - # Start CloudWatch agent /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s + -a fetch-config \ + -m ec2 \ + -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ + -s log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi - # Configure SSH access if public key provided if [ -n "" ]; then - log "Configuring SSH access" + log "Configuring SSH access" - # Determine the default user based on the home directory owner - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - # Create .ssh directory if it doesn't exist - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" + mkdir -p "/home/ec2-user/.ssh" + chmod 700 "/home/ec2-user/.ssh" - # Add the public key to authorized_keys - echo "" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" + echo "" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" - # Set proper ownership - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi - log "SSH key added for user $DEFAULT_USER" + log "SSH key added for user $DEFAULT_USER" fi log "Working directory: $homedir" @@ -1687,68 +1397,75 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 - # Detect architecture and download appropriate runner ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - # For ARM, replace x64 with arm64 in the URL - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" + RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" + RUNNER_URL="test.tar.gz" + log "x64 detected, using: $RUNNER_URL" fi curl -L $RUNNER_URL -o runner.tar.gz log "Extracting runner" - # `--no-overwrite-dir` is important, otherwise `$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$homedir` tar --no-overwrite-dir -xzf runner.tar.gz - # Create job tracking scripts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job started: ${GITHUB_JOB}" - mkdir -p /var/run/github-runner-jobs - echo '{"status":"running"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + mkdir -p $V-jobs + echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity $V-has-run-job EOFS cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 + V="/var/run/github-runner" echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job - touch /var/run/github-runner-last-activity + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' + cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh - A="/var/run/github-runner-last-activity" - J="/var/run/github-runner-jobs" - [ ! -f "$A" ] && touch "$A" - L=$(stat -c %Y "$A" 2>/dev/null || echo 0) - I=$(($(date +%s)-L)) - if ls $J/*.job 2>/dev/null | grep -q .; then G=${RUNNER_GRACE_PERIOD:-60}; else G=${RUNNER_INITIAL_GRACE_PERIOD:-180}; fi - R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) - if [ $R -eq 0 ] && [ $I -gt $G ]; then - echo "[$(date)] Terminating: idle $I > grace $G, proceeding with termination" - [ -f "$homedir/config.sh" ] && cd "$homedir" && pkill -INT -f "Runner.Listener" 2>/dev/null || true + V="/var/run/github-runner" + A="\$V-last-activity" + J="\$V-jobs" + H="\$V-has-run-job" + D="$homedir" + T="test" + + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" + if [ -f "\$D/config.sh" ]; then + cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true sleep 2 - if [ -f "$homedir/config.sh" ] && RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token test; then - echo "[$(date)] Runner removed from GitHub successfully" - fi - - flush_cloudwatch_logs - sudo shutdown -h now "Runner terminating after idle timeout" + log "Deregistering runner..." + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 + log "Deregistration exit: \$?" + else + log "ERROR: config.sh not found at \$D" + fi + flush_cloudwatch_logs + log "Shutting down" + sudo shutdown -h now else - echo "[$(date)] Activity detected within ${CURRENT_GRACE_PERIOD} seconds, not terminating" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - # Set up runner hooks echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env echo "RUNNER_HOME=$homedir" >> .env @@ -1756,13 +1473,11 @@ echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env echo "RUNNER_POLL_INTERVAL=11" >> .env - # Set up job tracking directory - mkdir -p /var/run/github-runner-jobs + V="/var/run/github-runner" + mkdir -p $V-jobs - # Create initial activity timestamp - touch /var/run/github-runner-last-activity + touch $V-last-activity - # Set up periodic termination check using systemd cat > /etc/systemd/system/runner-termination-check.service << 'EOF' [Unit] Description=Check GitHub runner termination conditions @@ -1786,84 +1501,65 @@ WantedBy=timers.target EOF - # Enable and start the timer systemctl daemon-reload systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - # Get instance metadata for descriptive runner name INSTANCE_ID=$(get_metadata "instance-id") INSTANCE_TYPE=$(get_metadata "instance-type") - # Create runner name with just the instance ID for uniqueness RUNNER_NAME="ec2-${INSTANCE_ID}" - # Build additional labels with metadata for easier correlation - # These will be visible in the GitHub runner management UI METADATA_LABELS="" METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - # Add GitHub workflow metadata passed from the action if [ -n "CI" ]; then - # Replace spaces and special chars in workflow name for label compatibility - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') + METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" fi if [ -n "42" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:42" + METADATA_LABELS="${METADATA_LABELS},run-number:42" fi - # Combine provided labels (user + runner-xxx) with instance metadata labels - # The label variable already contains user labels and the critical runner-xxx label from Python ALL_LABELS="label${METADATA_LABELS}" log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" log "Runner name: ${RUNNER_NAME}" log "Labels: ${ALL_LABELS}" - # Export token for error handler export RUNNER_TOKEN="test" - # Attempt to register. If a runner with the same name exists, config.sh will fail - # We use --unattended to prevent interactive prompts if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log_error "Failed to register runner '${RUNNER_NAME}'" + log_error "This usually means a runner with this name already exists" + log_error "If instance ID is 'unknown', metadata fetching likely failed" + terminate_instance "Runner registration failed" fi log "Runner registered successfully" - # Mark registration as complete and kill the watchdog log "Creating registration marker" touch /var/run/github-runner-registered ls -la /var/run/github-runner-registered - # Read watchdog PID from file (survives exec redirect) if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi - rm -f /var/run/github-runner-watchdog.pid + WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if kill -0 $WATCHDOG_PID 2>/dev/null; then + kill $WATCHDOG_PID 2>/dev/null || true + log "Killed registration watchdog (PID: $WATCHDOG_PID)" + else + log "Watchdog process $WATCHDOG_PID already terminated" + fi + rm -f /var/run/github-runner-watchdog.pid fi log "Starting runner" - # Create marker file for cleanup service touch /var/run/github-runner-started - # Ensure CloudWatch agent can read diagnostic logs - # The cwagent user needs to traverse into /home/ec2-user to reach _diag - # Make /home/ec2-user world-executable (but not readable) so cwagent can traverse it + chmod o+x $homedir log "Made $homedir traversable for CloudWatch agent" - # Create _diag directory if it doesn't exist mkdir -p $homedir/_diag - # The _diag files are already world-readable by default, just ensure the directory is too chmod 755 $homedir/_diag ./run.sh From 38aa70055eb2bec12247a75f02e408c864bea650 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 01:34:24 -0400 Subject: [PATCH 09/69] Implement multiple runners per instance support - Add runners_per_instance input to action.yml and runner.yml - Modify __main__.py to generate multiple tokens per instance - Update StartAWS to handle grouped tokens for multiple runners - Add runners_per_instance parameter to action.yml and runner.yml (default: 1) - Generate multiple GitHub runner tokens upfront for multi-runner instances - Update template to register and start multiple runners in separate directories - Each runner gets its own directory (runner-0, runner-1, etc.) - Update termination logic to handle all runners on instance - Maintain backward compatibility with single runner mode - Update output handling to provide array of all runner labels This allows a single EC2 instance to host multiple GitHub runners, enabling concurrent job execution without launching separate instances. --- .github/workflows/demo-dbg-minimal.yml | 17 -- .github/workflows/demo-gpu.yml | 74 ------ .github/workflows/demo-multi-runner.yml | 87 ++++++- .github/workflows/demos.yml | 3 + .github/workflows/runner.yml | 6 + action.yml | 4 + src/ec2_gha/__main__.py | 20 +- src/ec2_gha/start.py | 54 +++- src/ec2_gha/templates/user-script.sh.templ | 278 ++++++++++++--------- 9 files changed, 316 insertions(+), 227 deletions(-) delete mode 100644 .github/workflows/demo-dbg-minimal.yml delete mode 100644 .github/workflows/demo-gpu.yml diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml deleted file mode 100644 index 7b37cc4..0000000 --- a/.github/workflows/demo-dbg-minimal.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Demo – configurable instance for debugging -on: - workflow_dispatch: - workflow_call: # Tested by `demos.yml` -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout -jobs: - ec2: - # To use from another repo: Open-Athena/ec2-gha/.github/workflows/runner.yml - uses: ./.github/workflows/runner.yml - secrets: inherit - gpu-test: - needs: ec2 - runs-on: ${{ needs.ec2.outputs.id }} - steps: - - run: nvidia-smi # Verify GPU is available diff --git a/.github/workflows/demo-gpu.yml b/.github/workflows/demo-gpu.yml deleted file mode 100644 index 0a0cb0a..0000000 --- a/.github/workflows/demo-gpu.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Demo – toy GPU workload, optional sleep (for debugging) -on: - workflow_dispatch: - inputs: - sleep: - description: "Sleep for this many seconds before completing job (optional, helps keep instance alive for SSH access and debugging)" - required: false - type: number - default: 0 - ssh_pubkey: - description: "Add this SSH public key to instance's `~/.ssh/authorized_keys` (optional, for debugging)" - required: false - type: string - workflow_call: # Tested by `demos.yml` - inputs: - sleep: - required: false - type: number - default: 0 - ssh_pubkey: - required: false - type: string - -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout - -jobs: - ec2: - uses: ./.github/workflows/runner.yml - secrets: inherit - with: - ssh_pubkey: ${{ inputs.ssh_pubkey }} - - gpu-workload: - needs: ec2 - runs-on: ${{ needs.ec2.outputs.id }} - steps: - - uses: actions/checkout@v4 - - - name: System info - run: | - echo "=== System Info ===" - uname -a - lscpu | grep "Model name" || true - free -h - - echo -e "\n=== GPU Info ===" - nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv - - - name: Install PyTorch with CUDA support - run: | - echo "=== Installing PyTorch with CUDA support ===" - # Install python3-venv if not available - sudo apt-get update && sudo apt-get install -y python3-venv - # Use system Python3 and create a virtual environment to avoid pip root warning - python3 --version - python3 -m venv /tmp/venv - source /tmp/venv/bin/activate - pip install torch --index-url https://download.pytorch.org/whl/cu121 - echo "PyTorch installed successfully" - - - name: Run PyTorch GPU benchmark - run: | - echo "=== PyTorch GPU Benchmark ===" - source /tmp/venv/bin/activate - python .github/test-scripts/gpu-benchmark.py - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: | - echo "Sleeping for ${{ inputs.sleep }} seconds (useful for SSH debugging)..." - echo "Instance will remain available at the IP shown in the GitHub Actions log" - sleep ${{ inputs.sleep }} diff --git a/.github/workflows/demo-multi-runner.yml b/.github/workflows/demo-multi-runner.yml index b3e4b52..4ccac83 100644 --- a/.github/workflows/demo-multi-runner.yml +++ b/.github/workflows/demo-multi-runner.yml @@ -1,8 +1,87 @@ -name: Demo – multiple runners on one instance +name: Demo – multiple runners on single instance on: workflow_dispatch: + inputs: + runners_per_instance: + description: "Number of runners per EC2 instance" + required: false + type: string + default: "3" + sleep_duration: + description: "Sleep duration in seconds for each job" + required: false + type: string + default: "10" + workflow_call: # Tested by `demos.yml` + inputs: + runners_per_instance: + required: false + type: string + default: "3" + sleep_duration: + required: false + type: string + default: "10" + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout + jobs: - placeholder: - runs-on: ubuntu-latest + ec2: + name: Launch 1 EC2 instance with ${{ inputs.runners_per_instance }} runners + uses: ./.github/workflows/runner.yml + secrets: inherit + with: + instance_name: "$repo/$name (#$run_number) – $idx" + instance_count: "1" + runners_per_instance: ${{ inputs.runners_per_instance }} + ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) + # Use a larger instance type to handle multiple runners + ec2_instance_type: t3.xlarge # 4 vCPUs, 16 GB RAM + + parallel-jobs: + needs: ec2 + strategy: + matrix: + # Parse the JSON array of runner labels and use them in the matrix + runner: ${{ fromJson(needs.ec2.outputs.instances) }} + runs-on: ${{ matrix.runner }} + name: Job on ${{ matrix.runner }} steps: - - run: echo "Placeholder workflow – being developed in #2 / rw/hooks branch" + - uses: actions/checkout@v4 + + - name: Runner info + run: | + echo "Running on runner with label: ${{ matrix.runner }}" + echo "Hostname: $(hostname)" + echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" + echo "Region: $(curl -s http://169.254.169.254/latest/meta-data/placement/region)" + echo "Runner index: ${RUNNER_INDEX:-unknown}" + echo "Runner home: ${RUNNER_HOME:-unknown}" + + - name: Simulate workload + run: | + # All runners on same instance run independently + DURATION=${{ inputs.sleep_duration }} + echo "Starting at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" + echo "Simulating workload for ${DURATION} seconds..." + sleep $DURATION + echo "Workload complete at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" + + - name: Verify parallelism + run: | + echo "This job ran in parallel with other matrix jobs on the SAME instance" + echo "With ${{ inputs.runners_per_instance }} runners and ${{ inputs.sleep_duration }}s sleep:" + echo "- Sequential execution would take: $((${{ inputs.runners_per_instance }} * ${{ inputs.sleep_duration }}))s" + echo "- Parallel execution should take: ~${{ inputs.sleep_duration }}s (plus overhead)" + + - name: Show resource sharing + run: | + echo "=== Resource Usage ===" + echo "CPU cores available: $(nproc)" + echo "Memory available: $(free -h | grep Mem | awk '{print $2}')" + echo "Load average: $(uptime | awk -F'load average:' '{print $2}')" + echo "" + echo "Note: All ${{ inputs.runners_per_instance }} runners share these resources" diff --git a/.github/workflows/demos.yml b/.github/workflows/demos.yml index 52dc0a3..60498af 100644 --- a/.github/workflows/demos.yml +++ b/.github/workflows/demos.yml @@ -23,3 +23,6 @@ jobs: demo-multi-job: uses: ./.github/workflows/demo-multi-job.yml secrets: inherit + demo-multi-runner: + uses: ./.github/workflows/demo-multi-runner.yml + secrets: inherit diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index 270ad0b..938a224 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -105,6 +105,11 @@ on: description: "Maximum seconds to wait for runner to register with GitHub (falls back to vars.RUNNER_REGISTRATION_TIMEOUT, then 360 = 6 minutes)" required: false type: string + runners_per_instance: + description: "Number of runners to register per instance (each in separate directories to allow concurrent jobs)" + required: false + type: string + default: "1" ssh_pubkey: description: "SSH public key to add to authorized_keys (falls back to vars.SSH_PUBKEY)" required: false @@ -175,6 +180,7 @@ jobs: runner_initial_grace_period: ${{ inputs.runner_initial_grace_period || vars.RUNNER_INITIAL_GRACE_PERIOD }} runner_poll_interval: ${{ inputs.runner_poll_interval || vars.RUNNER_POLL_INTERVAL }} runner_registration_timeout: ${{ inputs.runner_registration_timeout || vars.RUNNER_REGISTRATION_TIMEOUT }} + runners_per_instance: ${{ inputs.runners_per_instance }} ssh_pubkey: ${{ inputs.ssh_pubkey || vars.SSH_PUBKEY }} env: GH_PAT: ${{ secrets.GH_SA_TOKEN }} diff --git a/action.yml b/action.yml index d81dd43..dac6dc3 100644 --- a/action.yml +++ b/action.yml @@ -69,6 +69,10 @@ inputs: runner_registration_timeout: description: "Maximum seconds to wait for runner to register with GitHub (falls back to vars.RUNNER_REGISTRATION_TIMEOUT, then 360 = 6 minutes)" required: false + runners_per_instance: + description: "Number of runners to register per instance (each in separate directories to allow concurrent jobs)" + required: false + default: "1" ssh_pubkey: description: "SSH public key to add to authorized_keys for debugging access" required: false diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index 60d6ad5..8362c8c 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -48,6 +48,7 @@ def main(): .update_state("INPUT_RUNNER_GRACE_PERIOD", "runner_grace_period") .update_state("INPUT_RUNNER_INITIAL_GRACE_PERIOD", "runner_initial_grace_period") .update_state("INPUT_RUNNER_POLL_INTERVAL", "runner_poll_interval") + .update_state("INPUT_RUNNERS_PER_INSTANCE", "runners_per_instance", type_hint=int) .update_state("INPUT_SSH_PUBKEY", "ssh_pubkey") .update_state("AWS_REGION", "region_name") # default .update_state("INPUT_AWS_REGION", "region_name") # input override @@ -61,8 +62,9 @@ def main(): if repo is None: raise Exception("Repo cannot be empty") - # Instance count is not a keyword arg for StartAWS, so we remove it + # Instance count and runners_per_instance are not keyword args for StartAWS, so we remove them instance_count = params.pop("instance_count", INSTANCE_COUNT) + runners_per_instance = params.pop("runners_per_instance", 1) # Apply defaults that weren't set via inputs or vars params.setdefault("max_instance_lifetime", MAX_INSTANCE_LIFETIME) @@ -79,6 +81,22 @@ def main(): # home_dir will be set to AUTO in start.py if not provided gh = GitHubInstance(token=token, repo=repo) + + # Pass runners_per_instance to StartAWS + params["runners_per_instance"] = runners_per_instance + + # Generate all the tokens we need upfront + # Each instance needs runners_per_instance tokens + total_runners = instance_count * runners_per_instance + if runners_per_instance > 1: + # Generate all tokens upfront + all_tokens = gh.create_runner_tokens(total_runners) + # Group tokens by instance (each instance gets runners_per_instance tokens) + grouped_tokens = [] + for i in range(0, total_runners, runners_per_instance): + grouped_tokens.append(all_tokens[i:i+runners_per_instance]) + params["grouped_runner_tokens"] = grouped_tokens + # This will create a new instance of StartAWS and configure it correctly deployment = DeployInstance( provider_type=StartAWS, diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 7622e8f..e7e1b7f 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -50,6 +50,8 @@ class StartAWS(CreateCloudInstance): Grace period in seconds before terminating instance after last job completes. Defaults to "60". runner_poll_interval : str How often (in seconds) to check termination conditions. Defaults to "10". + runners_per_instance : int + Number of runners to register per instance. Defaults to 1. script : str The script to run on the instance. Defaults to an empty string. security_group_id : str @@ -71,6 +73,7 @@ class StartAWS(CreateCloudInstance): repo: str cloudwatch_logs_group: str = "" gh_runner_tokens: list[str] = field(default_factory=list) + grouped_runner_tokens: list[list[str]] = field(default_factory=list) home_dir: str = "" iam_instance_profile: str = "" instance_name: str = "" @@ -81,6 +84,7 @@ class StartAWS(CreateCloudInstance): runner_grace_period: str = "60" runner_initial_grace_period: str = "180" runner_poll_interval: str = "10" + runners_per_instance: int = 1 runner_release: str = "" script: str = "" security_group_id: str = "" @@ -313,10 +317,26 @@ def create_instances(self) -> dict[str, str]: if not self.home_dir: self.home_dir = AUTO id_dict = {} - for idx, token in enumerate(self.gh_runner_tokens): - label = gh.GitHubInstance.generate_random_label() - # Combine user labels with the generated runner label - labels = f"{self.labels},{label}" if self.labels else label + # Determine which tokens to use + tokens_to_use = self.grouped_runner_tokens if self.grouped_runner_tokens else [[t] for t in self.gh_runner_tokens] + + for idx, instance_tokens in enumerate(tokens_to_use): + # Generate labels and tokens for all runners on this instance + runner_configs = [] + for runner_idx, token in enumerate(instance_tokens): + label = gh.GitHubInstance.generate_random_label() + # Combine user labels with the generated runner label + labels = f"{self.labels},{label}" if self.labels else label + runner_configs.append({ + "token": token, + "labels": labels, + "runner_idx": runner_idx + }) + + # Always provide runner_configs, even for single runner (backward compatibility) + # This simplifies the template logic + primary_labels = runner_configs[0]["labels"] if runner_configs else "" + base_token = instance_tokens[0] if instance_tokens else "" user_data_params = { "cloudwatch_logs_group": self.cloudwatch_logs_group, @@ -324,7 +344,7 @@ def create_instances(self) -> dict[str, str]: "github_run_id": environ.get("GITHUB_RUN_ID", ""), "github_run_number": environ.get("GITHUB_RUN_NUMBER", ""), "homedir": self.home_dir, - "labels": labels, + "labels": primary_labels, # Keep for backward compatibility "max_instance_lifetime": self.max_instance_lifetime, "repo": self.repo, "runner_grace_period": self.runner_grace_period, @@ -332,9 +352,12 @@ def create_instances(self) -> dict[str, str]: "runner_poll_interval": self.runner_poll_interval, "runner_registration_timeout": environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "").strip() or RUNNER_REGISTRATION_TIMEOUT, "runner_release": self.runner_release, + "runners_per_instance": str(self.runners_per_instance), + # Base64 encode the JSON to avoid shell escaping issues + "runner_configs_b64": __import__('base64').b64encode(json.dumps(runner_configs).encode()).decode(), "script": self.script, "ssh_pubkey": self.ssh_pubkey, - "token": token, + "token": base_token, # Keep for backward compatibility "userdata": self.userdata, } params = self._build_aws_params(user_data_params, idx=idx) @@ -362,7 +385,13 @@ def create_instances(self) -> dict[str, str]: raise instances = result["Instances"] id = instances[0]["InstanceId"] - id_dict[id] = label + # For multiple runners per instance, store all labels + if self.runners_per_instance > 1: + all_labels = [config["labels"] for config in runner_configs] + id_dict[id] = all_labels + else: + # For backward compatibility, store single label as string + id_dict[id] = primary_labels return id_dict def wait_until_ready(self, ids: list[str], **kwargs): @@ -398,12 +427,19 @@ def set_instance_mapping(self, mapping: dict[str, str]): A dictionary of instance IDs and labels. """ - github_labels = list(mapping.values()) + # Flatten all labels from all instances + github_labels = [] + for labels in mapping.values(): + if isinstance(labels, list): + github_labels.extend(labels) + else: + github_labels.append(labels) + output("mapping", json.dumps(mapping)) output("instances", json.dumps(github_labels)) # For single instance use, output simplified values - if len(mapping) == 1: + if len(mapping) == 1 and self.runners_per_instance == 1: instance_id = list(mapping.keys())[0] label = list(mapping.values())[0] output("instance-id", instance_id) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 1406d9a..faad6e5 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -1,5 +1,10 @@ #!/bin/bash set -e + +# Enable debug tracing to a file for troubleshooting +exec 2> >(tee -a /var/log/runner-debug.log >&2) +set -x + log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $$1" >&2; } @@ -36,17 +41,25 @@ get_metadata() { else curl -s "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" fi + return 0 # Always return success to avoid set -e issues } logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR -INSTANCE_ID=$$(get_metadata "instance-id") - terminate_instance() { local reason="$$1" - log "FATAL: $$reason" - log "Terminating instance $$INSTANCE_ID due to setup failure" + local instance_id=$$(get_metadata "instance-id") + + # Log error prominently + echo "========================================" | tee -a /var/log/runner-setup.log + log "FATAL ERROR DETECTED" + log "Reason: $$reason" + log "Instance: $$instance_id" + log "Script location: $$(pwd)" + log "User: $$(whoami)" + log "Debug trace available in: /var/log/runner-debug.log" + echo "========================================" | tee -a /var/log/runner-setup.log # Try to remove runner if it was partially configured if [ -f "$$homedir/config.sh" ] && [ -n "$${RUNNER_TOKEN:-}" ]; then @@ -54,6 +67,14 @@ terminate_instance() { fi flush_cloudwatch_logs + + # Give time for debugging + log "Sleeping for 300 seconds before shutdown to allow debugging..." + log "SSH into instance with: ssh ubuntu@$$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 300 + + log "Shutting down after debug delay" shutdown -h now exit 1 } @@ -124,16 +145,16 @@ else log "Using: $$homedir" fi -# Fetch instance metadata before redirecting output +# Redirect runner setup logs to a file for CloudWatch +exec >> /var/log/runner-setup.log 2>&1 +log "Starting runner setup" + +# Fetch instance metadata INSTANCE_TYPE=$$(get_metadata "instance-type") INSTANCE_ID=$$(get_metadata "instance-id") REGION=$$(get_metadata "placement/region") AZ=$$(get_metadata "placement/availability-zone") -# Redirect runner setup logs to a file for CloudWatch -exec >> /var/log/runner-setup.log 2>&1 -log "Starting runner setup" - log "Instance metadata: Type=$${INSTANCE_TYPE} ID=$${INSTANCE_ID} Region=$${REGION} AZ=$${AZ}" # Set up maximum lifetime timeout - do this early to ensure cleanup @@ -176,6 +197,7 @@ if [ "$cloudwatch_logs_group" != "" ]; then "files": { "collect_list": [ { "file_path": "/var/log/runner-setup.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/var/log/runner-debug.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, { "file_path": "/tmp/job-started-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, { "file_path": "/tmp/termination-check.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, @@ -229,28 +251,30 @@ echo "$script" > pre-runner-script.sh log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 -# Detect architecture and download appropriate runner + +# Runner configurations are always provided as JSON from Python +RUNNERS_PER_INSTANCE=$runners_per_instance + +# Download runner binary once ARCH=$$(uname -m) if [ "$$ARCH" = "aarch64" ] || [ "$$ARCH" = "arm64" ]; then - # For ARM, replace x64 with arm64 in the URL RUNNER_URL=$$(echo "$runner_release" | sed 's/x64/arm64/g') log "ARM detected, using: $$RUNNER_URL" else RUNNER_URL="$runner_release" log "x64 detected, using: $$RUNNER_URL" fi -curl -L $$RUNNER_URL -o runner.tar.gz -log "Extracting runner" -# `--no-overwrite-dir` is important, otherwise `$$homedir` ends up `chown`'d to `1001:docker`, and `sshd` will refuse connection attempts to `$$homedir` -tar --no-overwrite-dir -xzf runner.tar.gz -# Create job tracking scripts +curl -L $$RUNNER_URL -o /tmp/runner.tar.gz +log "Downloaded runner binary" +# Create shared job tracking scripts (used by all runners) cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 V="/var/run/github-runner" -echo "[$$(date)] ${log_prefix_job_started} $${GITHUB_JOB}" +RUNNER_IDX="$${RUNNER_INDEX:-0}" +echo "[$$(date)] Runner-$$RUNNER_IDX: ${log_prefix_job_started} $${GITHUB_JOB}" mkdir -p $$V-jobs -echo '{"status":"running"}' > $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job +echo '{"status":"running","runner":"'$$RUNNER_IDX'"}' > $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job touch $$V-last-activity $$V-has-run-job EOFS @@ -258,63 +282,58 @@ cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 V="/var/run/github-runner" -echo "[$$(date)] ${log_prefix_job_completed} $${GITHUB_JOB}" -rm -f $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}.job +RUNNER_IDX="$${RUNNER_INDEX:-0}" +echo "[$$(date)] Runner-$$RUNNER_IDX: ${log_prefix_job_completed} $${GITHUB_JOB}" +rm -f $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job touch $$V-last-activity EOFC -cat > /usr/local/bin/check-runner-termination.sh << EOFT +cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' #!/bin/bash exec >> /tmp/termination-check.log 2>&1 source /usr/local/bin/runner-common-functions.sh V="/var/run/github-runner" -A="\$$V-last-activity" -J="\$$V-jobs" -H="\$$V-has-run-job" -D="$$homedir" -T="$token" - -[ ! -f "\$$A" ] && touch "\$$A" -L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) -N=\$$(date +%s) -I=\$$((N-L)) -[ -f "\$$H" ] && G=\$${RUNNER_GRACE_PERIOD:-60} || G=\$${RUNNER_INITIAL_GRACE_PERIOD:-180} -R=\$$(grep -l '"status":"running"' \$$J/*.job 2>/dev/null | wc -l || echo 0) - -if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then - log "TERMINATING: idle \$$I > grace \$$G" - if [ -f "\$$D/config.sh" ]; then - cd "\$$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true - sleep 2 - log "Deregistering runner..." - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$$T 2>&1 - log "Deregistration exit: \$$?" - else - log "ERROR: config.sh not found at \$$D" - fi +A="$$V-last-activity" +J="$$V-jobs" +H="$$V-has-run-job" + +[ ! -f "$$A" ] && touch "$$A" +L=$$(stat -c %Y "$$A" 2>/dev/null || echo 0) +N=$$(date +%s) +I=$$((N-L)) +[ -f "$$H" ] && G=$${RUNNER_GRACE_PERIOD:-60} || G=$${RUNNER_INITIAL_GRACE_PERIOD:-180} +R=$$(grep -l '"status":"running"' $$J/*.job 2>/dev/null | wc -l || echo 0) + +if [ $$R -eq 0 ] && [ $$I -gt $$G ]; then + log "TERMINATING: idle $$I > grace $$G" + # Deregister all runners + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "$$RUNNER_DIR" ] && [ -f "$$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in $$RUNNER_DIR" + cd "$$RUNNER_DIR" + pkill -INT -f "$$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + # Read token from stored file + if [ -f "$$RUNNER_DIR/.runner-token" ]; then + TOKEN=$$(cat "$$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $$TOKEN 2>&1 + log "Deregistration exit: $$?" + fi + fi + done flush_cloudwatch_logs log "Shutting down" sudo shutdown -h now else - [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" + [ $$R -gt 0 ] && log "$$R job(s) running" || log "Idle $$I/$$G sec" fi EOFT chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh -# Set up runner hooks -echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env -echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env -echo "RUNNER_HOME=$$homedir" >> .env -echo "RUNNER_GRACE_PERIOD=$runner_grace_period" >> .env -echo "RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" >> .env -echo "RUNNER_POLL_INTERVAL=$runner_poll_interval" >> .env - # Set up job tracking directory V="/var/run/github-runner" mkdir -p $$V-jobs - -# Create initial activity timestamp touch $$V-last-activity # Set up periodic termination check using systemd @@ -322,7 +341,6 @@ cat > /etc/systemd/system/runner-termination-check.service << 'EOF' [Unit] Description=Check GitHub runner termination conditions After=network.target - [Service] Type=oneshot ExecStart=/usr/local/bin/check-runner-termination.sh @@ -332,93 +350,109 @@ cat > /etc/systemd/system/runner-termination-check.timer << EOF [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service - [Timer] OnBootSec=60s OnUnitActiveSec=${runner_poll_interval}s - [Install] WantedBy=timers.target EOF -# Enable and start the timer systemctl daemon-reload systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer -# Get instance metadata for descriptive runner name -INSTANCE_ID=$$(get_metadata "instance-id") -INSTANCE_TYPE=$$(get_metadata "instance-type") - -# Create runner name with just the instance ID for uniqueness -RUNNER_NAME="ec2-$${INSTANCE_ID}" - -# Build additional labels with metadata for easier correlation -# These will be visible in the GitHub runner management UI -METADATA_LABELS="" -METADATA_LABELS="$${METADATA_LABELS},instance-id:$${INSTANCE_ID}" -METADATA_LABELS="$${METADATA_LABELS},instance-type:$${INSTANCE_TYPE}" - -# Add GitHub workflow metadata passed from the action +# Build metadata labels (using metadata fetched earlier) +METADATA_LABELS=",instance-id:$${INSTANCE_ID},instance-type:$${INSTANCE_TYPE}" if [ -n "$github_workflow" ]; then - # Replace spaces and special chars in workflow name for label compatibility WORKFLOW_LABEL=$$(echo "$github_workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="$${METADATA_LABELS},workflow:$${WORKFLOW_LABEL}" fi -if [ -n "$github_run_id" ]; then - METADATA_LABELS="$${METADATA_LABELS},run-id:$github_run_id" -fi -if [ -n "$github_run_number" ]; then - METADATA_LABELS="$${METADATA_LABELS},run-number:$github_run_number" -fi - -# Combine provided labels (user + runner-xxx) with instance metadata labels -# The $labels variable already contains user labels and the critical runner-xxx label from Python -ALL_LABELS="$labels$${METADATA_LABELS}" - -log "Configuring runner for repo: $repo" -log "Runner name: $${RUNNER_NAME}" -log "Labels: $${ALL_LABELS}" -# Export token for error handler -export RUNNER_TOKEN="$token" - -# Attempt to register. If a runner with the same name exists, config.sh will fail -# We use --unattended to prevent interactive prompts -if ! ./config.sh --url https://github.com/$repo --token $token --labels "$${ALL_LABELS}" --name "$${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '$${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" +[ -n "$github_run_id" ] && METADATA_LABELS="$${METADATA_LABELS},run-id:$github_run_id" +[ -n "$github_run_number" ] && METADATA_LABELS="$${METADATA_LABELS},run-number:$github_run_number" + +# Process each runner configuration +log "Setting up $$RUNNERS_PER_INSTANCE runner(s)" +python3 -c " +import json, sys, os, subprocess, traceback, base64 + +# Runner configurations passed from Python (base64 encoded to avoid shell escaping issues) +runner_configs_b64 = '$runner_configs_b64' +runner_configs_json = base64.b64decode(runner_configs_b64).decode() + +try: + configs = json.loads(runner_configs_json) +except Exception as e: + print(f'ERROR: Failed to parse runner configs: {e}', file=sys.stderr) + print(f'Input was: {runner_configs_json}', file=sys.stderr) + traceback.print_exc() + sys.exit(1) +homedir = '$$homedir' # Use shell variable (already resolved from AUTO) +repo = '$repo' +instance_id = '$$INSTANCE_ID' +metadata_labels = '$$METADATA_LABELS' + +print(f'Processing {len(configs)} runner configuration(s)', file=sys.stderr) +for config in configs: + idx = config['runner_idx'] + token = config['token'] + labels = config['labels'] + metadata_labels + + print(f'Configuring runner {idx}...', file=sys.stderr) + + # Create runner directory + runner_dir = f'{homedir}/runner-{idx}' + os.makedirs(runner_dir, exist_ok=True) + os.chdir(runner_dir) + + # Extract runner + subprocess.run(['tar', '-xzf', '/tmp/runner.tar.gz'], check=True) + + # Save token for deregistration + with open('.runner-token', 'w') as f: + f.write(token) + + # Create env file + with open('.env', 'w') as f: + f.write(f'ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh\\n') + f.write(f'ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh\\n') + f.write(f'RUNNER_HOME={runner_dir}\\n') + f.write(f'RUNNER_INDEX={idx}\\n') + f.write(f'RUNNER_GRACE_PERIOD=$runner_grace_period\\n') + f.write(f'RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period\\n') + + # Configure runner + runner_name = f'ec2-{instance_id}-{idx}' + cmd = ['./config.sh', '--url', f'https://github.com/{repo}', '--token', token, + '--labels', labels, '--name', runner_name, '--disableupdate', '--unattended'] + result = subprocess.run(cmd, env={'RUNNER_ALLOW_RUNASROOT': '1'}) + if result.returncode != 0: + print(f'Failed to register runner {idx}') + sys.exit(1) + + # Start runner in background + # Inherit system environment and add RUNNER_ALLOW_RUNASROOT + runner_env = os.environ.copy() + runner_env['RUNNER_ALLOW_RUNASROOT'] = '1' + subprocess.Popen(['./run.sh'], env=runner_env) + print(f'Started runner {idx} in {runner_dir}') +" + +if [ $$? -ne 0 ]; then + terminate_instance "Failed to register runners" fi -log "Runner registered successfully" -# Mark registration as complete and kill the watchdog -log "Creating registration marker" +log "All runners registered and started" touch /var/run/github-runner-registered -ls -la /var/run/github-runner-registered -# Read watchdog PID from file (survives exec redirect) + +# Kill watchdog if [ -f /var/run/github-runner-watchdog.pid ]; then WATCHDOG_PID=$$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $$WATCHDOG_PID 2>/dev/null; then - kill $$WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $$WATCHDOG_PID)" - else - log "Watchdog process $$WATCHDOG_PID already terminated" - fi + kill $$WATCHDOG_PID 2>/dev/null || true rm -f /var/run/github-runner-watchdog.pid fi -log "Starting runner" -# Create marker file for cleanup service touch /var/run/github-runner-started - -# Ensure CloudWatch agent can read diagnostic logs -# The cwagent user needs to traverse into $homedir to reach _diag -# Make $homedir world-executable (but not readable) so cwagent can traverse it chmod o+x $$homedir -log "Made $$homedir traversable for CloudWatch agent" -# Create _diag directory if it doesn't exist -mkdir -p $$homedir/_diag -# The _diag files are already world-readable by default, just ensure the directory is too -chmod 755 $$homedir/_diag -./run.sh +for RUNNER_DIR in $$homedir/runner-*; do + [ -d "$$RUNNER_DIR/_diag" ] && chmod 755 "$$RUNNER_DIR/_diag" +done From ad60f7c450f5493d1cb5653dfab074deec02fa85 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 14:42:01 -0400 Subject: [PATCH 10/69] Fix runner grace period environment variables in systemd service The RUNNER_INITIAL_GRACE_PERIOD and RUNNER_GRACE_PERIOD values were being correctly passed to the instance and set in the runner's .env file, but the systemd service that runs the termination check didn't have access to these environment variables. Added Environment= directives to the systemd service to pass the grace period values through, so they'll be available when the termination check script runs. --- src/ec2_gha/__main__.py | 1 + src/ec2_gha/templates/user-script.sh.templ | 28 ++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index 8362c8c..e348f93 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -13,6 +13,7 @@ from gha_runner.clouddeployment import DeployInstance from gha_runner.helper.input import EnvVarBuilder, check_required import os +from os import environ def main(): diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index faad6e5..776bae7 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -337,12 +337,14 @@ mkdir -p $$V-jobs touch $$V-last-activity # Set up periodic termination check using systemd -cat > /etc/systemd/system/runner-termination-check.service << 'EOF' +cat > /etc/systemd/system/runner-termination-check.service << EOF [Unit] Description=Check GitHub runner termination conditions After=network.target [Service] Type=oneshot +Environment="RUNNER_GRACE_PERIOD=$runner_grace_period" +Environment="RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" ExecStart=/usr/local/bin/check-runner-termination.sh EOF @@ -424,17 +426,29 @@ for config in configs: runner_name = f'ec2-{instance_id}-{idx}' cmd = ['./config.sh', '--url', f'https://github.com/{repo}', '--token', token, '--labels', labels, '--name', runner_name, '--disableupdate', '--unattended'] - result = subprocess.run(cmd, env={'RUNNER_ALLOW_RUNASROOT': '1'}) - if result.returncode != 0: - print(f'Failed to register runner {idx}') + result = subprocess.run(cmd, env={'RUNNER_ALLOW_RUNASROOT': '1'}, capture_output=True, text=True) + + # Check if registration succeeded (even if config.sh returns non-zero) + if 'Runner successfully added' in result.stdout: + print(f'Runner {idx} registered successfully', file=sys.stderr) + elif result.returncode != 0: + print(f'Failed to register runner {idx}: {result.stderr}', file=sys.stderr) sys.exit(1) - # Start runner in background # Inherit system environment and add RUNNER_ALLOW_RUNASROOT runner_env = os.environ.copy() runner_env['RUNNER_ALLOW_RUNASROOT'] = '1' - subprocess.Popen(['./run.sh'], env=runner_env) - print(f'Started runner {idx} in {runner_dir}') + # Start runner in background (use full path and handle errors) + try: + proc = subprocess.Popen([f'{runner_dir}/run.sh'], + env=runner_env, + cwd=runner_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + print(f'Started runner {idx} in {runner_dir} (PID: {proc.pid})', file=sys.stderr) + except Exception as e: + print(f'Failed to start runner {idx}: {e}', file=sys.stderr) + sys.exit(1) " if [ $$? -ne 0 ]; then From 641aaca27b72bbdec3f152b3a397ca8e8209ced6 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 15:00:32 -0400 Subject: [PATCH 11/69] Fix runner deregistration by properly substituting homedir path The termination check script was failing to deregister runners because $homedir was inside a single-quoted heredoc ('EOFT'), preventing variable substitution. This caused the script to look for /runner-* instead of /home/ubuntu/runner-*. Fixed by: - Changing heredoc delimiter from 'EOFT' to EOFT (allows substitution) - Escaping shell variables with \ to preserve them for runtime - Ensuring $homedir is substituted at template generation time This ensures runners are properly deregistered before instance termination, preventing orphaned runners in GitHub. --- src/ec2_gha/templates/user-script.sh.templ | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 776bae7..bbaba81 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -288,36 +288,36 @@ rm -f $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job touch $$V-last-activity EOFC -cat > /usr/local/bin/check-runner-termination.sh << 'EOFT' +cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 source /usr/local/bin/runner-common-functions.sh V="/var/run/github-runner" -A="$$V-last-activity" -J="$$V-jobs" -H="$$V-has-run-job" - -[ ! -f "$$A" ] && touch "$$A" -L=$$(stat -c %Y "$$A" 2>/dev/null || echo 0) -N=$$(date +%s) -I=$$((N-L)) -[ -f "$$H" ] && G=$${RUNNER_GRACE_PERIOD:-60} || G=$${RUNNER_INITIAL_GRACE_PERIOD:-180} -R=$$(grep -l '"status":"running"' $$J/*.job 2>/dev/null | wc -l || echo 0) - -if [ $$R -eq 0 ] && [ $$I -gt $$G ]; then - log "TERMINATING: idle $$I > grace $$G" +A="\$$V-last-activity" +J="\$$V-jobs" +H="\$$V-has-run-job" + +[ ! -f "\$$A" ] && touch "\$$A" +L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) +N=\$$(date +%s) +I=\$$((N-L)) +[ -f "\$$H" ] && G=\$${RUNNER_GRACE_PERIOD:-60} || G=\$${RUNNER_INITIAL_GRACE_PERIOD:-180} +R=\$$(grep -l '"status":"running"' \$$J/*.job 2>/dev/null | wc -l || echo 0) + +if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then + log "TERMINATING: idle \$$I > grace \$$G" # Deregister all runners - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "$$RUNNER_DIR" ] && [ -f "$$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in $$RUNNER_DIR" - cd "$$RUNNER_DIR" - pkill -INT -f "$$RUNNER_DIR/run.sh" 2>/dev/null || true + for RUNNER_DIR in $$homedir/runner-*; do + if [ -d "\$$RUNNER_DIR" ] && [ -f "\$$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in \$$RUNNER_DIR" + cd "\$$RUNNER_DIR" + pkill -INT -f "\$$RUNNER_DIR/run.sh" 2>/dev/null || true sleep 1 # Read token from stored file - if [ -f "$$RUNNER_DIR/.runner-token" ]; then - TOKEN=$$(cat "$$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $$TOKEN 2>&1 - log "Deregistration exit: $$?" + if [ -f "\$$RUNNER_DIR/.runner-token" ]; then + TOKEN=\$$(cat "\$$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$$TOKEN 2>&1 + log "Deregistration exit: \$$?" fi fi done @@ -325,7 +325,7 @@ if [ $$R -eq 0 ] && [ $$I -gt $$G ]; then log "Shutting down" sudo shutdown -h now else - [ $$R -gt 0 ] && log "$$R job(s) running" || log "Idle $$I/$$G sec" + [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" fi EOFT From 0fee210969bc0100da7241d6c7ff6f61bf5be814 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 15:12:22 -0400 Subject: [PATCH 12/69] Add optional debug mode for troubleshooting Added a 'debug' input parameter that enables verbose output (set -x) in the runner setup script. This helps with troubleshooting without always having verbose output. Changes: - Added 'debug' input to action.yml and runner.yml workflow - Pass debug parameter through Python code to template - Conditionally enable 'set -x' only when debug is true - Keep runner-debug.log output regardless (useful for post-mortem) The debug mode can be enabled by setting debug: true in the workflow that calls ec2-gha. --- .github/workflows/runner.yml | 6 ++++++ action.yml | 3 +++ src/ec2_gha/__main__.py | 1 + src/ec2_gha/start.py | 2 ++ src/ec2_gha/templates/user-script.sh.templ | 7 ++++++- 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index 938a224..fd4c17e 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -38,6 +38,11 @@ on: description: "CloudWatch Logs group name for streaming runner logs (leave empty to disable)" required: false type: string + debug: + description: "Enable debug output (set -x) in runner setup script" + required: false + type: boolean + default: false ec2_home_dir: description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR, then auto-detection)" required: false @@ -165,6 +170,7 @@ jobs: aws_region: ${{ inputs.aws_region || vars.AWS_REGION }} aws_tags: ${{ inputs.aws_tags }} cloudwatch_logs_group: ${{ inputs.cloudwatch_logs_group || vars.CLOUDWATCH_LOGS_GROUP }} + debug: ${{ inputs.debug }} ec2_home_dir: ${{ inputs.ec2_home_dir || vars.EC2_HOME_DIR }} ec2_image_id: ${{ inputs.ec2_image_id || vars.EC2_IMAGE_ID }} ec2_instance_profile: ${{ inputs.ec2_instance_profile || vars.EC2_INSTANCE_PROFILE }} diff --git a/action.yml b/action.yml index dac6dc3..4d26906 100644 --- a/action.yml +++ b/action.yml @@ -16,6 +16,9 @@ inputs: cloudwatch_logs_group: description: "CloudWatch Logs group name for streaming runner logs (leave empty to disable)" required: false + debug: + description: "Enable debug output (set -x) in runner setup script" + required: false ec2_home_dir: description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR, then auto-detection)" required: false diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index e348f93..a7b506a 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -34,6 +34,7 @@ def main(): .update_state("INPUT_AWS_SUBNET_ID", "subnet_id") .update_state("INPUT_AWS_TAGS", "tags", is_json=True) .update_state("INPUT_CLOUDWATCH_LOGS_GROUP", "cloudwatch_logs_group") + .update_state("INPUT_DEBUG", "debug") .update_state("INPUT_EC2_HOME_DIR", "home_dir") .update_state("INPUT_EC2_IMAGE_ID", "image_id") .update_state("INPUT_EC2_INSTANCE_PROFILE", "iam_instance_profile") diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index e7e1b7f..eb06ae1 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -72,6 +72,7 @@ class StartAWS(CreateCloudInstance): region_name: str repo: str cloudwatch_logs_group: str = "" + debug: str = "" gh_runner_tokens: list[str] = field(default_factory=list) grouped_runner_tokens: list[list[str]] = field(default_factory=list) home_dir: str = "" @@ -340,6 +341,7 @@ def create_instances(self) -> dict[str, str]: user_data_params = { "cloudwatch_logs_group": self.cloudwatch_logs_group, + "debug": self.debug, "github_workflow": environ.get("GITHUB_WORKFLOW", ""), "github_run_id": environ.get("GITHUB_RUN_ID", ""), "github_run_number": environ.get("GITHUB_RUN_NUMBER", ""), diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index bbaba81..73c2555 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -3,7 +3,12 @@ set -e # Enable debug tracing to a file for troubleshooting exec 2> >(tee -a /var/log/runner-debug.log >&2) -set -x + +# Conditionally enable debug mode +if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + echo "[DEBUG] Debug mode enabled - set -x active" >&2 + set -x +fi log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $$1" >&2; } From ee915ceaf640f3d716c16d545edadc00f2d4b350 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 15:15:43 -0400 Subject: [PATCH 13/69] Make system dependencies more robust and cross-platform Added checks for critical system dependencies and made the script more portable across different Linux distributions: - Check for Python3 availability (critical dependency) - Support both dpkg (Debian/Ubuntu) and rpm (RHEL/CentOS/Amazon Linux) for CloudWatch agent installation - Support both curl and wget for downloading files - Fail gracefully with clear error messages when critical tools are missing This makes ec2-gha more flexible for use with different AMIs beyond just Ubuntu, while maintaining backward compatibility. --- src/ec2_gha/templates/user-script.sh.templ | 65 ++++++++++++++++------ 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 73c2555..35d9745 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -173,23 +173,36 @@ if [ "$cloudwatch_logs_group" != "" ]; then # Use a subshell to prevent CloudWatch failures from stopping the entire script ( - # Wait for dpkg lock to be released (up to 2 minutes) - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $$timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($$timeout seconds remaining)" - sleep 5 - timeout=$$((timeout - 5)) - done - - # Download and install CloudWatch agent - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb + # Detect package manager and install CloudWatch agent + if command -v dpkg >/dev/null 2>&1; then + # Debian/Ubuntu + log "Detected dpkg-based system" + + # Wait for dpkg lock to be released (up to 2 minutes) + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $$timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($$timeout seconds remaining)" + sleep 5 + timeout=$$((timeout - 5)) + done + + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + # RHEL/CentOS/Amazon Linux + log "Detected rpm-based system" + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + else + log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" + fi # Configure CloudWatch agent cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF @@ -269,7 +282,16 @@ else RUNNER_URL="$runner_release" log "x64 detected, using: $$RUNNER_URL" fi -curl -L $$RUNNER_URL -o /tmp/runner.tar.gz + +# Check for curl or wget +if command -v curl >/dev/null 2>&1; then + curl -L $$RUNNER_URL -o /tmp/runner.tar.gz +elif command -v wget >/dev/null 2>&1; then + wget -q $$RUNNER_URL -O /tmp/runner.tar.gz +else + log_error "Neither curl nor wget found. Cannot download runner." + terminate_instance "No download tool available" +fi log "Downloaded runner binary" # Create shared job tracking scripts (used by all runners) cat > /usr/local/bin/job-started-hook.sh << 'EOFS' @@ -379,6 +401,13 @@ fi # Process each runner configuration log "Setting up $$RUNNERS_PER_INSTANCE runner(s)" + +# Check for Python3 availability +if ! command -v python3 >/dev/null 2>&1; then + log_error "Python3 is required but not found. Cannot configure runners." + terminate_instance "Python3 not available on this AMI" +fi + python3 -c " import json, sys, os, subprocess, traceback, base64 From ed21f506ac6f095959048b2766ee4e1ef6a7903a Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 15:21:25 -0400 Subject: [PATCH 14/69] Parallelize runner setup and add Python3-free fallback Major improvements to runner configuration: 1. Parallelization (when Python3 available): - Moved runner setup logic to a shell function 'configure_runner' - Python now uses ThreadPoolExecutor to configure up to 4 runners in parallel - Significantly reduces setup time for multiple runners 2. Python3-free fallback: - Single runner can now be configured without Python3 - Uses shell-based JSON parsing (basic sed extraction) - Limited to single runner due to JSON parsing complexity in pure shell - Allows ec2-gha to work on minimal AMIs without Python3 3. Better separation of concerns: - Shell function handles all runner setup logic - Python only handles JSON parsing and parallelization - Makes the code more maintainable and testable The default path uses Python3 for reliability and performance, but the fallback ensures basic functionality on minimal AMIs. --- .github/workflows/demo-multi-runner.yml | 12 +- src/ec2_gha/start.py | 16 +- src/ec2_gha/templates/user-script.sh.templ | 192 ++-- tests/__snapshots__/test_start.ambr | 1148 +++++++++++++------- tests/test_start.py | 6 +- 5 files changed, 907 insertions(+), 467 deletions(-) diff --git a/.github/workflows/demo-multi-runner.yml b/.github/workflows/demo-multi-runner.yml index 4ccac83..c6d2c17 100644 --- a/.github/workflows/demo-multi-runner.yml +++ b/.github/workflows/demo-multi-runner.yml @@ -7,7 +7,7 @@ on: required: false type: string default: "3" - sleep_duration: + sleep: description: "Sleep duration in seconds for each job" required: false type: string @@ -18,7 +18,7 @@ on: required: false type: string default: "3" - sleep_duration: + sleep: required: false type: string default: "10" @@ -64,7 +64,7 @@ jobs: - name: Simulate workload run: | # All runners on same instance run independently - DURATION=${{ inputs.sleep_duration }} + DURATION=${{ inputs.sleep }} echo "Starting at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" echo "Simulating workload for ${DURATION} seconds..." sleep $DURATION @@ -73,9 +73,9 @@ jobs: - name: Verify parallelism run: | echo "This job ran in parallel with other matrix jobs on the SAME instance" - echo "With ${{ inputs.runners_per_instance }} runners and ${{ inputs.sleep_duration }}s sleep:" - echo "- Sequential execution would take: $((${{ inputs.runners_per_instance }} * ${{ inputs.sleep_duration }}))s" - echo "- Parallel execution should take: ~${{ inputs.sleep_duration }}s (plus overhead)" + echo "With ${{ inputs.runners_per_instance }} runners and ${{ inputs.sleep }}s sleep:" + echo "- Sequential execution would take: $((${{ inputs.runners_per_instance }} * ${{ inputs.sleep }}))s" + echo "- Parallel execution should take: ~${{ inputs.sleep }}s (plus overhead)" - name: Show resource sharing run: | diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index eb06ae1..17b43a1 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -334,10 +334,10 @@ def create_instances(self) -> dict[str, str]: "runner_idx": runner_idx }) - # Always provide runner_configs, even for single runner (backward compatibility) - # This simplifies the template logic - primary_labels = runner_configs[0]["labels"] if runner_configs else "" - base_token = instance_tokens[0] if instance_tokens else "" + # Simplify runner configs to save template space + # Pass tokens as space-delimited, labels as pipe-delimited + runner_tokens = " ".join(config["token"] for config in runner_configs) + runner_labels = "|".join(config["labels"] for config in runner_configs) user_data_params = { "cloudwatch_logs_group": self.cloudwatch_logs_group, @@ -346,7 +346,6 @@ def create_instances(self) -> dict[str, str]: "github_run_id": environ.get("GITHUB_RUN_ID", ""), "github_run_number": environ.get("GITHUB_RUN_NUMBER", ""), "homedir": self.home_dir, - "labels": primary_labels, # Keep for backward compatibility "max_instance_lifetime": self.max_instance_lifetime, "repo": self.repo, "runner_grace_period": self.runner_grace_period, @@ -355,11 +354,10 @@ def create_instances(self) -> dict[str, str]: "runner_registration_timeout": environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "").strip() or RUNNER_REGISTRATION_TIMEOUT, "runner_release": self.runner_release, "runners_per_instance": str(self.runners_per_instance), - # Base64 encode the JSON to avoid shell escaping issues - "runner_configs_b64": __import__('base64').b64encode(json.dumps(runner_configs).encode()).decode(), + "runner_tokens": runner_tokens, # Space-delimited tokens + "runner_labels": runner_labels, # Pipe-delimited labels "script": self.script, "ssh_pubkey": self.ssh_pubkey, - "token": base_token, # Keep for backward compatibility "userdata": self.userdata, } params = self._build_aws_params(user_data_params, idx=idx) @@ -393,7 +391,7 @@ def create_instances(self) -> dict[str, str]: id_dict[id] = all_labels else: # For backward compatibility, store single label as string - id_dict[id] = primary_labels + id_dict[id] = runner_configs[0]["labels"] if runner_configs else "" return id_dict def wait_until_ready(self, ids: list[str], **kwargs): diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 35d9745..f78eca4 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -402,91 +402,115 @@ fi # Process each runner configuration log "Setting up $$RUNNERS_PER_INSTANCE runner(s)" -# Check for Python3 availability -if ! command -v python3 >/dev/null 2>&1; then - log_error "Python3 is required but not found. Cannot configure runners." - terminate_instance "Python3 not available on this AMI" -fi +# Function to configure a single runner +# Export it so it's available to subprocesses +configure_runner() { + local idx=$$1 + local token=$$2 + local labels=$$3 + local homedir=$$4 + local repo=$$5 + local instance_id=$$6 + local runner_grace_period=$$7 + local runner_initial_grace_period=$$8 + + log "Configuring runner $$idx..." + + # Create runner directory + local runner_dir="$$homedir/runner-$$idx" + mkdir -p "$$runner_dir" + cd "$$runner_dir" + + # Extract runner + tar -xzf /tmp/runner.tar.gz + + # Save token for deregistration + echo "$$token" > .runner-token + + # Create env file + cat > .env << EOF +ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh +ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh +RUNNER_HOME=$$runner_dir +RUNNER_INDEX=$$idx +RUNNER_GRACE_PERIOD=$$runner_grace_period +RUNNER_INITIAL_GRACE_PERIOD=$$runner_initial_grace_period +EOF + + # Configure runner + local runner_name="ec2-$$instance_id-$$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ + --url "https://github.com/$$repo" \ + --token "$$token" \ + --labels "$$labels" \ + --name "$$runner_name" \ + --disableupdate \ + --unattended 2>&1 | tee /tmp/runner-$$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$$idx-config.log; then + log "Runner $$idx registered successfully" + else + log_error "Failed to register runner $$idx" + return 1 + fi + + # Start runner in background + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$$! + log "Started runner $$idx in $$runner_dir (PID: $$pid)" + + return 0 +} +# Export the function so it's available to child processes +export -f configure_runner +export -f log +export -f log_error + +# Parse simple delimited format: space-delimited tokens, pipe-delimited labels +IFS=' ' read -ra tokens <<< "$runner_tokens" +IFS='|' read -ra labels <<< "$runner_labels" + +num_runners=$${#tokens[@]} +log "Configuring $$num_runners runner(s) in parallel" + +# Start configuration for each runner in background +pids=() +for i in $${!tokens[@]}; do + token=$${tokens[$$i]} + label=$${labels[$$i]:-} # Default to empty if no label + + if [ -z "$$token" ]; then + log_error "No token for runner $$i" + continue + fi + + # Start configuration in background + ( + configure_runner $$i "$$token" "$${label}$$METADATA_LABELS" "$$homedir" "$repo" "$$INSTANCE_ID" "$runner_grace_period" "$runner_initial_grace_period" + echo $$? > /tmp/runner-$$i-status + ) & + pids+=($$!) + + log "Started configuration for runner $$i (PID: $${pids[-1]})" +done + +# Wait for all background jobs to complete +log "Waiting for all runner configurations to complete..." +failed=0 +for i in $${!pids[@]}; do + wait $${pids[$$i]} + if [ -f /tmp/runner-$$i-status ]; then + status=$$(cat /tmp/runner-$$i-status) + rm -f /tmp/runner-$$i-status + if [ "$$status" != "0" ]; then + log_error "Runner $$i configuration failed" + failed=1 + fi + fi +done -python3 -c " -import json, sys, os, subprocess, traceback, base64 - -# Runner configurations passed from Python (base64 encoded to avoid shell escaping issues) -runner_configs_b64 = '$runner_configs_b64' -runner_configs_json = base64.b64decode(runner_configs_b64).decode() - -try: - configs = json.loads(runner_configs_json) -except Exception as e: - print(f'ERROR: Failed to parse runner configs: {e}', file=sys.stderr) - print(f'Input was: {runner_configs_json}', file=sys.stderr) - traceback.print_exc() - sys.exit(1) -homedir = '$$homedir' # Use shell variable (already resolved from AUTO) -repo = '$repo' -instance_id = '$$INSTANCE_ID' -metadata_labels = '$$METADATA_LABELS' - -print(f'Processing {len(configs)} runner configuration(s)', file=sys.stderr) -for config in configs: - idx = config['runner_idx'] - token = config['token'] - labels = config['labels'] + metadata_labels - - print(f'Configuring runner {idx}...', file=sys.stderr) - - # Create runner directory - runner_dir = f'{homedir}/runner-{idx}' - os.makedirs(runner_dir, exist_ok=True) - os.chdir(runner_dir) - - # Extract runner - subprocess.run(['tar', '-xzf', '/tmp/runner.tar.gz'], check=True) - - # Save token for deregistration - with open('.runner-token', 'w') as f: - f.write(token) - - # Create env file - with open('.env', 'w') as f: - f.write(f'ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh\\n') - f.write(f'ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh\\n') - f.write(f'RUNNER_HOME={runner_dir}\\n') - f.write(f'RUNNER_INDEX={idx}\\n') - f.write(f'RUNNER_GRACE_PERIOD=$runner_grace_period\\n') - f.write(f'RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period\\n') - - # Configure runner - runner_name = f'ec2-{instance_id}-{idx}' - cmd = ['./config.sh', '--url', f'https://github.com/{repo}', '--token', token, - '--labels', labels, '--name', runner_name, '--disableupdate', '--unattended'] - result = subprocess.run(cmd, env={'RUNNER_ALLOW_RUNASROOT': '1'}, capture_output=True, text=True) - - # Check if registration succeeded (even if config.sh returns non-zero) - if 'Runner successfully added' in result.stdout: - print(f'Runner {idx} registered successfully', file=sys.stderr) - elif result.returncode != 0: - print(f'Failed to register runner {idx}: {result.stderr}', file=sys.stderr) - sys.exit(1) - - # Inherit system environment and add RUNNER_ALLOW_RUNASROOT - runner_env = os.environ.copy() - runner_env['RUNNER_ALLOW_RUNASROOT'] = '1' - # Start runner in background (use full path and handle errors) - try: - proc = subprocess.Popen([f'{runner_dir}/run.sh'], - env=runner_env, - cwd=runner_dir, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - print(f'Started runner {idx} in {runner_dir} (PID: {proc.pid})', file=sys.stderr) - except Exception as e: - print(f'Failed to start runner {idx}: {e}', file=sys.stderr) - sys.exit(1) -" - -if [ $$? -ne 0 ]; then - terminate_instance "Failed to register runners" +if [ $$failed -ne 0 ]; then + terminate_instance "One or more runners failed to register" fi log "All runners registered and started" diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index dc3becd..edabb8a 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -43,6 +43,14 @@ 'UserData': ''' #!/bin/bash set -e + + exec 2> >(tee -a /var/log/runner-debug.log >&2) + + if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then + echo "[DEBUG] Debug mode enabled - set -x active" >&2 + set -x + fi + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -76,23 +84,37 @@ else curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" fi + return 0 # Always return success to avoid set -e issues } logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - INSTANCE_ID=$(get_metadata "instance-id") - terminate_instance() { local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local instance_id=$(get_metadata "instance-id") + + echo "========================================" | tee -a /var/log/runner-setup.log + log "FATAL ERROR DETECTED" + log "Reason: $reason" + log "Instance: $instance_id" + log "Script location: $(pwd)" + log "User: $(whoami)" + log "Debug trace available in: /var/log/runner-debug.log" + echo "========================================" | tee -a /var/log/runner-setup.log if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true fi flush_cloudwatch_logs + + log "Sleeping for 300 seconds before shutdown to allow debugging..." + log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 300 + + log "Shutting down after debug delay" shutdown -h now exit 1 } @@ -152,14 +174,14 @@ log "Using: $homedir" fi + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -170,21 +192,32 @@ log "Installing CloudWatch agent" ( - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done - - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb + if command -v dpkg >/dev/null 2>&1; then + log "Detected dpkg-based system" + + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done + + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + log "Detected rpm-based system" + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + else + log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" + fi cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { @@ -196,6 +229,7 @@ "files": { "collect_list": [ { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/var/log/runner-debug.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, @@ -243,6 +277,9 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 + + RUNNERS_PER_INSTANCE=1 + ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') @@ -251,16 +288,24 @@ RUNNER_URL="test.tar.gz" log "x64 detected, using: $RUNNER_URL" fi - curl -L $RUNNER_URL -o runner.tar.gz - log "Extracting runner" - tar --no-overwrite-dir -xzf runner.tar.gz + + if command -v curl >/dev/null 2>&1; then + curl -L $RUNNER_URL -o /tmp/runner.tar.gz + elif command -v wget >/dev/null 2>&1; then + wget -q $RUNNER_URL -O /tmp/runner.tar.gz + else + log_error "Neither curl nor wget found. Cannot download runner." + terminate_instance "No download tool available" + fi + log "Downloaded runner binary" cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job started: ${GITHUB_JOB}" + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -268,8 +313,9 @@ #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity EOFC @@ -281,8 +327,6 @@ A="\$V-last-activity" J="\$V-jobs" H="\$V-has-run-job" - D="$homedir" - T="test" [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) @@ -293,15 +337,19 @@ if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then log "TERMINATING: idle \$I > grace \$G" - if [ -f "\$D/config.sh" ]; then - cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true - sleep 2 - log "Deregistering runner..." - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 - log "Deregistration exit: \$?" - else - log "ERROR: config.sh not found at \$D" - fi + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in \$RUNNER_DIR" + cd "\$RUNNER_DIR" + pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "\$RUNNER_DIR/.runner-token" ]; then + TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 + log "Deregistration exit: \$?" + fi + fi + done flush_cloudwatch_logs log "Shutting down" sudo shutdown -h now @@ -312,25 +360,18 @@ chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env - echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env - echo "RUNNER_HOME=$homedir" >> .env - echo "RUNNER_GRACE_PERIOD=61" >> .env - echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env - echo "RUNNER_POLL_INTERVAL=11" >> .env - V="/var/run/github-runner" mkdir -p $V-jobs - touch $V-last-activity - cat > /etc/systemd/system/runner-termination-check.service << 'EOF' + cat > /etc/systemd/system/runner-termination-check.service << EOF [Unit] Description=Check GitHub runner termination conditions After=network.target - [Service] Type=oneshot + Environment="RUNNER_GRACE_PERIOD=61" + Environment="RUNNER_INITIAL_GRACE_PERIOD=181" ExecStart=/usr/local/bin/check-runner-termination.sh EOF @@ -338,11 +379,9 @@ [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service - [Timer] OnBootSec=60s OnUnitActiveSec=11s - [Install] WantedBy=timers.target EOF @@ -351,63 +390,128 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - INSTANCE_ID=$(get_metadata "instance-id") - INSTANCE_TYPE=$(get_metadata "instance-type") - - RUNNER_NAME="ec2-${INSTANCE_ID}" - - METADATA_LABELS="" - METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" - METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - + METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" if [ -n "CI" ]; then WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi - if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - fi - if [ -n "1" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:1" - fi + [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + [ -n "1" ] && METADATA_LABELS="${METADATA_LABELS},run-number:1" + + log "Setting up $RUNNERS_PER_INSTANCE runner(s)" + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 + + log "Configuring runner $idx..." + + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + + tar -xzf /tmp/runner.tar.gz + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF + + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ + --url "https://github.com/$repo" \ + --token "$token" \ + --labels "$labels" \ + --name "$runner_name" \ + --disableupdate \ + --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi - ALL_LABELS="label${METADATA_LABELS}" + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" - log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" - log "Runner name: ${RUNNER_NAME}" - log "Labels: ${ALL_LABELS}" - export RUNNER_TOKEN="test" + return 0 + } + export -f configure_runner + export -f log + export -f log_error + + IFS=' ' read -ra tokens <<< "test" + IFS='|' read -ra labels <<< "label" + + num_runners=${#tokens[@]} + log "Configuring $num_runners runner(s) in parallel" + + pids=() + for i in ${!tokens[@]}; do + token=${tokens[$i]} + label=${labels[$i]:-} # Default to empty if no label + + if [ -z "$token" ]; then + log_error "No token for runner $i" + continue + fi + + ( + configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" + echo $? > /tmp/runner-$i-status + ) & + pids+=($!) + + log "Started configuration for runner $i (PID: ${pids[-1]})" + done - if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log "Waiting for all runner configurations to complete..." + failed=0 + for i in ${!pids[@]}; do + wait ${pids[$i]} + if [ -f /tmp/runner-$i-status ]; then + status=$(cat /tmp/runner-$i-status) + rm -f /tmp/runner-$i-status + if [ "$status" != "0" ]; then + log_error "Runner $i configuration failed" + failed=1 + fi + fi + done + + if [ $failed -ne 0 ]; then + terminate_instance "One or more runners failed to register" fi - log "Runner registered successfully" - log "Creating registration marker" + log "All runners registered and started" touch /var/run/github-runner-registered - ls -la /var/run/github-runner-registered + if [ -f /var/run/github-runner-watchdog.pid ]; then WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi + kill $WATCHDOG_PID 2>/dev/null || true rm -f /var/run/github-runner-watchdog.pid fi - log "Starting runner" touch /var/run/github-runner-started - chmod o+x $homedir - log "Made $homedir traversable for CloudWatch agent" - mkdir -p $homedir/_diag - chmod 755 $homedir/_diag - ./run.sh + for RUNNER_DIR in $homedir/runner-*; do + [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" + done ''', }) @@ -452,6 +556,14 @@ 'UserData': ''' #!/bin/bash set -e + + exec 2> >(tee -a /var/log/runner-debug.log >&2) + + if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then + echo "[DEBUG] Debug mode enabled - set -x active" >&2 + set -x + fi + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -485,23 +597,37 @@ else curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" fi + return 0 # Always return success to avoid set -e issues } logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - INSTANCE_ID=$(get_metadata "instance-id") - terminate_instance() { local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local instance_id=$(get_metadata "instance-id") + + echo "========================================" | tee -a /var/log/runner-setup.log + log "FATAL ERROR DETECTED" + log "Reason: $reason" + log "Instance: $instance_id" + log "Script location: $(pwd)" + log "User: $(whoami)" + log "Debug trace available in: /var/log/runner-debug.log" + echo "========================================" | tee -a /var/log/runner-setup.log if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true fi flush_cloudwatch_logs + + log "Sleeping for 300 seconds before shutdown to allow debugging..." + log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 300 + + log "Shutting down after debug delay" shutdown -h now exit 1 } @@ -561,14 +687,14 @@ log "Using: $homedir" fi + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -579,21 +705,32 @@ log "Installing CloudWatch agent" ( - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done - - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb + if command -v dpkg >/dev/null 2>&1; then + log "Detected dpkg-based system" + + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done + + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + log "Detected rpm-based system" + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + else + log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" + fi cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { @@ -605,6 +742,7 @@ "files": { "collect_list": [ { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/var/log/runner-debug.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, @@ -652,6 +790,9 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 + + RUNNERS_PER_INSTANCE=1 + ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') @@ -660,16 +801,24 @@ RUNNER_URL="test.tar.gz" log "x64 detected, using: $RUNNER_URL" fi - curl -L $RUNNER_URL -o runner.tar.gz - log "Extracting runner" - tar --no-overwrite-dir -xzf runner.tar.gz + + if command -v curl >/dev/null 2>&1; then + curl -L $RUNNER_URL -o /tmp/runner.tar.gz + elif command -v wget >/dev/null 2>&1; then + wget -q $RUNNER_URL -O /tmp/runner.tar.gz + else + log_error "Neither curl nor wget found. Cannot download runner." + terminate_instance "No download tool available" + fi + log "Downloaded runner binary" cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job started: ${GITHUB_JOB}" + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -677,8 +826,9 @@ #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity EOFC @@ -690,8 +840,6 @@ A="\$V-last-activity" J="\$V-jobs" H="\$V-has-run-job" - D="$homedir" - T="test" [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) @@ -702,15 +850,19 @@ if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then log "TERMINATING: idle \$I > grace \$G" - if [ -f "\$D/config.sh" ]; then - cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true - sleep 2 - log "Deregistering runner..." - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 - log "Deregistration exit: \$?" - else - log "ERROR: config.sh not found at \$D" - fi + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in \$RUNNER_DIR" + cd "\$RUNNER_DIR" + pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "\$RUNNER_DIR/.runner-token" ]; then + TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 + log "Deregistration exit: \$?" + fi + fi + done flush_cloudwatch_logs log "Shutting down" sudo shutdown -h now @@ -721,25 +873,18 @@ chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env - echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env - echo "RUNNER_HOME=$homedir" >> .env - echo "RUNNER_GRACE_PERIOD=61" >> .env - echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env - echo "RUNNER_POLL_INTERVAL=11" >> .env - V="/var/run/github-runner" mkdir -p $V-jobs - touch $V-last-activity - cat > /etc/systemd/system/runner-termination-check.service << 'EOF' + cat > /etc/systemd/system/runner-termination-check.service << EOF [Unit] Description=Check GitHub runner termination conditions After=network.target - [Service] Type=oneshot + Environment="RUNNER_GRACE_PERIOD=61" + Environment="RUNNER_INITIAL_GRACE_PERIOD=181" ExecStart=/usr/local/bin/check-runner-termination.sh EOF @@ -747,11 +892,9 @@ [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service - [Timer] OnBootSec=60s OnUnitActiveSec=11s - [Install] WantedBy=timers.target EOF @@ -760,63 +903,128 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - INSTANCE_ID=$(get_metadata "instance-id") - INSTANCE_TYPE=$(get_metadata "instance-type") - - RUNNER_NAME="ec2-${INSTANCE_ID}" - - METADATA_LABELS="" - METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" - METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - + METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" if [ -n "CI" ]; then WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi - if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - fi - if [ -n "42" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:42" - fi + [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + [ -n "42" ] && METADATA_LABELS="${METADATA_LABELS},run-number:42" + + log "Setting up $RUNNERS_PER_INSTANCE runner(s)" + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 + + log "Configuring runner $idx..." + + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + + tar -xzf /tmp/runner.tar.gz + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF + + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ + --url "https://github.com/$repo" \ + --token "$token" \ + --labels "$labels" \ + --name "$runner_name" \ + --disableupdate \ + --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi - ALL_LABELS="label${METADATA_LABELS}" + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" - log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" - log "Runner name: ${RUNNER_NAME}" - log "Labels: ${ALL_LABELS}" - export RUNNER_TOKEN="test" + return 0 + } + export -f configure_runner + export -f log + export -f log_error + + IFS=' ' read -ra tokens <<< "test" + IFS='|' read -ra labels <<< "label" + + num_runners=${#tokens[@]} + log "Configuring $num_runners runner(s) in parallel" + + pids=() + for i in ${!tokens[@]}; do + token=${tokens[$i]} + label=${labels[$i]:-} # Default to empty if no label + + if [ -z "$token" ]; then + log_error "No token for runner $i" + continue + fi + + ( + configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" + echo $? > /tmp/runner-$i-status + ) & + pids+=($!) + + log "Started configuration for runner $i (PID: ${pids[-1]})" + done - if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + log "Waiting for all runner configurations to complete..." + failed=0 + for i in ${!pids[@]}; do + wait ${pids[$i]} + if [ -f /tmp/runner-$i-status ]; then + status=$(cat /tmp/runner-$i-status) + rm -f /tmp/runner-$i-status + if [ "$status" != "0" ]; then + log_error "Runner $i configuration failed" + failed=1 + fi + fi + done + + if [ $failed -ne 0 ]; then + terminate_instance "One or more runners failed to register" fi - log "Runner registered successfully" - log "Creating registration marker" + log "All runners registered and started" touch /var/run/github-runner-registered - ls -la /var/run/github-runner-registered + if [ -f /var/run/github-runner-watchdog.pid ]; then WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi + kill $WATCHDOG_PID 2>/dev/null || true rm -f /var/run/github-runner-watchdog.pid fi - log "Starting runner" touch /var/run/github-runner-started - chmod o+x $homedir - log "Made $homedir traversable for CloudWatch agent" - mkdir -p $homedir/_diag - chmod 755 $homedir/_diag - ./run.sh + for RUNNER_DIR in $homedir/runner-*; do + [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" + done ''', }) @@ -825,6 +1033,14 @@ ''' #!/bin/bash set -e + + exec 2> >(tee -a /var/log/runner-debug.log >&2) + + if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then + echo "[DEBUG] Debug mode enabled - set -x active" >&2 + set -x + fi + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -858,23 +1074,37 @@ else curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" fi + return 0 # Always return success to avoid set -e issues } logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - INSTANCE_ID=$(get_metadata "instance-id") - terminate_instance() { local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local instance_id=$(get_metadata "instance-id") + + echo "========================================" | tee -a /var/log/runner-setup.log + log "FATAL ERROR DETECTED" + log "Reason: $reason" + log "Instance: $instance_id" + log "Script location: $(pwd)" + log "User: $(whoami)" + log "Debug trace available in: /var/log/runner-debug.log" + echo "========================================" | tee -a /var/log/runner-setup.log if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true fi flush_cloudwatch_logs + + log "Sleeping for 300 seconds before shutdown to allow debugging..." + log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 300 + + log "Shutting down after debug delay" shutdown -h now exit 1 } @@ -934,14 +1164,14 @@ log "Using: $homedir" fi + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -952,21 +1182,32 @@ log "Installing CloudWatch agent" ( - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done + if command -v dpkg >/dev/null 2>&1; then + log "Detected dpkg-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done + + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + log "Detected rpm-based system" + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + else + log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" + fi cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { @@ -978,6 +1219,7 @@ "files": { "collect_list": [ { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/var/log/runner-debug.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, @@ -1025,6 +1267,9 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 + + RUNNERS_PER_INSTANCE=1 + ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') @@ -1033,16 +1278,24 @@ RUNNER_URL="test.tar.gz" log "x64 detected, using: $RUNNER_URL" fi - curl -L $RUNNER_URL -o runner.tar.gz - log "Extracting runner" - tar --no-overwrite-dir -xzf runner.tar.gz + + if command -v curl >/dev/null 2>&1; then + curl -L $RUNNER_URL -o /tmp/runner.tar.gz + elif command -v wget >/dev/null 2>&1; then + wget -q $RUNNER_URL -O /tmp/runner.tar.gz + else + log_error "Neither curl nor wget found. Cannot download runner." + terminate_instance "No download tool available" + fi + log "Downloaded runner binary" cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job started: ${GITHUB_JOB}" + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -1050,8 +1303,9 @@ #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity EOFC @@ -1063,8 +1317,6 @@ A="\$V-last-activity" J="\$V-jobs" H="\$V-has-run-job" - D="$homedir" - T="test" [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) @@ -1075,15 +1327,19 @@ if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then log "TERMINATING: idle \$I > grace \$G" - if [ -f "\$D/config.sh" ]; then - cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true - sleep 2 - log "Deregistering runner..." - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 - log "Deregistration exit: \$?" - else - log "ERROR: config.sh not found at \$D" - fi + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in \$RUNNER_DIR" + cd "\$RUNNER_DIR" + pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "\$RUNNER_DIR/.runner-token" ]; then + TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 + log "Deregistration exit: \$?" + fi + fi + done flush_cloudwatch_logs log "Shutting down" sudo shutdown -h now @@ -1094,25 +1350,18 @@ chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env - echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env - echo "RUNNER_HOME=$homedir" >> .env - echo "RUNNER_GRACE_PERIOD=61" >> .env - echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env - echo "RUNNER_POLL_INTERVAL=11" >> .env - V="/var/run/github-runner" mkdir -p $V-jobs - touch $V-last-activity - cat > /etc/systemd/system/runner-termination-check.service << 'EOF' + cat > /etc/systemd/system/runner-termination-check.service << EOF [Unit] Description=Check GitHub runner termination conditions After=network.target - [Service] Type=oneshot + Environment="RUNNER_GRACE_PERIOD=61" + Environment="RUNNER_INITIAL_GRACE_PERIOD=181" ExecStart=/usr/local/bin/check-runner-termination.sh EOF @@ -1120,11 +1369,9 @@ [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service - [Timer] OnBootSec=60s OnUnitActiveSec=11s - [Install] WantedBy=timers.target EOF @@ -1133,63 +1380,128 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - INSTANCE_ID=$(get_metadata "instance-id") - INSTANCE_TYPE=$(get_metadata "instance-type") - - RUNNER_NAME="ec2-${INSTANCE_ID}" - - METADATA_LABELS="" - METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" - METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - + METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" if [ -n "CI" ]; then WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi - if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - fi - if [ -n "42" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:42" - fi + [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + [ -n "42" ] && METADATA_LABELS="${METADATA_LABELS},run-number:42" + + log "Setting up $RUNNERS_PER_INSTANCE runner(s)" + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 + + log "Configuring runner $idx..." + + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + + tar -xzf /tmp/runner.tar.gz + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF - ALL_LABELS="label${METADATA_LABELS}" + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ + --url "https://github.com/$repo" \ + --token "$token" \ + --labels "$labels" \ + --name "$runner_name" \ + --disableupdate \ + --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi - log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" - log "Runner name: ${RUNNER_NAME}" - log "Labels: ${ALL_LABELS}" - export RUNNER_TOKEN="test" + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" - if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + return 0 + } + export -f configure_runner + export -f log + export -f log_error + + IFS=' ' read -ra tokens <<< "test" + IFS='|' read -ra labels <<< "label" + + num_runners=${#tokens[@]} + log "Configuring $num_runners runner(s) in parallel" + + pids=() + for i in ${!tokens[@]}; do + token=${tokens[$i]} + label=${labels[$i]:-} # Default to empty if no label + + if [ -z "$token" ]; then + log_error "No token for runner $i" + continue + fi + + ( + configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" + echo $? > /tmp/runner-$i-status + ) & + pids+=($!) + + log "Started configuration for runner $i (PID: ${pids[-1]})" + done + + log "Waiting for all runner configurations to complete..." + failed=0 + for i in ${!pids[@]}; do + wait ${pids[$i]} + if [ -f /tmp/runner-$i-status ]; then + status=$(cat /tmp/runner-$i-status) + rm -f /tmp/runner-$i-status + if [ "$status" != "0" ]; then + log_error "Runner $i configuration failed" + failed=1 + fi + fi + done + + if [ $failed -ne 0 ]; then + terminate_instance "One or more runners failed to register" fi - log "Runner registered successfully" - log "Creating registration marker" + log "All runners registered and started" touch /var/run/github-runner-registered - ls -la /var/run/github-runner-registered + if [ -f /var/run/github-runner-watchdog.pid ]; then WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi + kill $WATCHDOG_PID 2>/dev/null || true rm -f /var/run/github-runner-watchdog.pid fi - log "Starting runner" touch /var/run/github-runner-started - chmod o+x $homedir - log "Made $homedir traversable for CloudWatch agent" - mkdir -p $homedir/_diag - chmod 755 $homedir/_diag - ./run.sh + for RUNNER_DIR in $homedir/runner-*; do + [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" + done ''' # --- @@ -1197,6 +1509,14 @@ ''' #!/bin/bash set -e + + exec 2> >(tee -a /var/log/runner-debug.log >&2) + + if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then + echo "[DEBUG] Debug mode enabled - set -x active" >&2 + set -x + fi + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -1230,23 +1550,37 @@ else curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" fi + return 0 # Always return success to avoid set -e issues } logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - INSTANCE_ID=$(get_metadata "instance-id") - terminate_instance() { local reason="$1" - log "FATAL: $reason" - log "Terminating instance $INSTANCE_ID due to setup failure" + local instance_id=$(get_metadata "instance-id") + + echo "========================================" | tee -a /var/log/runner-setup.log + log "FATAL ERROR DETECTED" + log "Reason: $reason" + log "Instance: $instance_id" + log "Script location: $(pwd)" + log "User: $(whoami)" + log "Debug trace available in: /var/log/runner-debug.log" + echo "========================================" | tee -a /var/log/runner-setup.log if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true fi flush_cloudwatch_logs + + log "Sleeping for 300 seconds before shutdown to allow debugging..." + log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 300 + + log "Shutting down after debug delay" shutdown -h now exit 1 } @@ -1306,14 +1640,14 @@ log "Using: $homedir" fi + exec >> /var/log/runner-setup.log 2>&1 + log "Starting runner setup" + INSTANCE_TYPE=$(get_metadata "instance-type") INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -1324,21 +1658,32 @@ log "Installing CloudWatch agent" ( - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done + if command -v dpkg >/dev/null 2>&1; then + log "Detected dpkg-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb + log "Waiting for dpkg lock to be released..." + timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done + + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + log "Detected rpm-based system" + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + else + log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" + fi cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF { @@ -1350,6 +1695,7 @@ "files": { "collect_list": [ { "file_path": "/var/log/runner-setup.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, + { "file_path": "/var/log/runner-debug.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, { "file_path": "/tmp/job-started-hook.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, { "file_path": "/tmp/termination-check.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, @@ -1397,6 +1743,9 @@ log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 + + RUNNERS_PER_INSTANCE=1 + ARCH=$(uname -m) if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') @@ -1405,16 +1754,24 @@ RUNNER_URL="test.tar.gz" log "x64 detected, using: $RUNNER_URL" fi - curl -L $RUNNER_URL -o runner.tar.gz - log "Extracting runner" - tar --no-overwrite-dir -xzf runner.tar.gz + + if command -v curl >/dev/null 2>&1; then + curl -L $RUNNER_URL -o /tmp/runner.tar.gz + elif command -v wget >/dev/null 2>&1; then + wget -q $RUNNER_URL -O /tmp/runner.tar.gz + else + log_error "Neither curl nor wget found. Cannot download runner." + terminate_instance "No download tool available" + fi + log "Downloaded runner binary" cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job started: ${GITHUB_JOB}" + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -1422,8 +1779,9 @@ #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 V="/var/run/github-runner" - echo "[$(date)] Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}.job + RUNNER_IDX="${RUNNER_INDEX:-0}" + echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" + rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity EOFC @@ -1435,8 +1793,6 @@ A="\$V-last-activity" J="\$V-jobs" H="\$V-has-run-job" - D="$homedir" - T="test" [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) @@ -1447,15 +1803,19 @@ if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then log "TERMINATING: idle \$I > grace \$G" - if [ -f "\$D/config.sh" ]; then - cd "\$D" && pkill -INT -f "Runner.Listener" 2>/dev/null || true - sleep 2 - log "Deregistering runner..." - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$T 2>&1 - log "Deregistration exit: \$?" - else - log "ERROR: config.sh not found at \$D" - fi + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in \$RUNNER_DIR" + cd "\$RUNNER_DIR" + pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "\$RUNNER_DIR/.runner-token" ]; then + TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 + log "Deregistration exit: \$?" + fi + fi + done flush_cloudwatch_logs log "Shutting down" sudo shutdown -h now @@ -1466,25 +1826,18 @@ chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh - echo "ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh" > .env - echo "ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh" >> .env - echo "RUNNER_HOME=$homedir" >> .env - echo "RUNNER_GRACE_PERIOD=61" >> .env - echo "RUNNER_INITIAL_GRACE_PERIOD=181" >> .env - echo "RUNNER_POLL_INTERVAL=11" >> .env - V="/var/run/github-runner" mkdir -p $V-jobs - touch $V-last-activity - cat > /etc/systemd/system/runner-termination-check.service << 'EOF' + cat > /etc/systemd/system/runner-termination-check.service << EOF [Unit] Description=Check GitHub runner termination conditions After=network.target - [Service] Type=oneshot + Environment="RUNNER_GRACE_PERIOD=61" + Environment="RUNNER_INITIAL_GRACE_PERIOD=181" ExecStart=/usr/local/bin/check-runner-termination.sh EOF @@ -1492,11 +1845,9 @@ [Unit] Description=Periodic GitHub runner termination check Requires=runner-termination-check.service - [Timer] OnBootSec=60s OnUnitActiveSec=11s - [Install] WantedBy=timers.target EOF @@ -1505,63 +1856,128 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - INSTANCE_ID=$(get_metadata "instance-id") - INSTANCE_TYPE=$(get_metadata "instance-type") - - RUNNER_NAME="ec2-${INSTANCE_ID}" - - METADATA_LABELS="" - METADATA_LABELS="${METADATA_LABELS},instance-id:${INSTANCE_ID}" - METADATA_LABELS="${METADATA_LABELS},instance-type:${INSTANCE_TYPE}" - + METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" if [ -n "CI" ]; then WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" fi - if [ -n "16725250800" ]; then - METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - fi - if [ -n "42" ]; then - METADATA_LABELS="${METADATA_LABELS},run-number:42" - fi + [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" + [ -n "42" ] && METADATA_LABELS="${METADATA_LABELS},run-number:42" + + log "Setting up $RUNNERS_PER_INSTANCE runner(s)" + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 + + log "Configuring runner $idx..." + + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + + tar -xzf /tmp/runner.tar.gz + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF - ALL_LABELS="label${METADATA_LABELS}" + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ + --url "https://github.com/$repo" \ + --token "$token" \ + --labels "$labels" \ + --name "$runner_name" \ + --disableupdate \ + --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi - log "Configuring runner for repo: omsf-eco-infra/awsinfratesting" - log "Runner name: ${RUNNER_NAME}" - log "Labels: ${ALL_LABELS}" - export RUNNER_TOKEN="test" + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" - if ! ./config.sh --url https://github.com/omsf-eco-infra/awsinfratesting --token test --labels "${ALL_LABELS}" --name "${RUNNER_NAME}" --disableupdate --unattended; then - log_error "Failed to register runner '${RUNNER_NAME}'" - log_error "This usually means a runner with this name already exists" - log_error "If instance ID is 'unknown', metadata fetching likely failed" - terminate_instance "Runner registration failed" + return 0 + } + export -f configure_runner + export -f log + export -f log_error + + IFS=' ' read -ra tokens <<< "test" + IFS='|' read -ra labels <<< "label" + + num_runners=${#tokens[@]} + log "Configuring $num_runners runner(s) in parallel" + + pids=() + for i in ${!tokens[@]}; do + token=${tokens[$i]} + label=${labels[$i]:-} # Default to empty if no label + + if [ -z "$token" ]; then + log_error "No token for runner $i" + continue + fi + + ( + configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" + echo $? > /tmp/runner-$i-status + ) & + pids+=($!) + + log "Started configuration for runner $i (PID: ${pids[-1]})" + done + + log "Waiting for all runner configurations to complete..." + failed=0 + for i in ${!pids[@]}; do + wait ${pids[$i]} + if [ -f /tmp/runner-$i-status ]; then + status=$(cat /tmp/runner-$i-status) + rm -f /tmp/runner-$i-status + if [ "$status" != "0" ]; then + log_error "Runner $i configuration failed" + failed=1 + fi + fi + done + + if [ $failed -ne 0 ]; then + terminate_instance "One or more runners failed to register" fi - log "Runner registered successfully" - log "Creating registration marker" + log "All runners registered and started" touch /var/run/github-runner-registered - ls -la /var/run/github-runner-registered + if [ -f /var/run/github-runner-watchdog.pid ]; then WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) - if kill -0 $WATCHDOG_PID 2>/dev/null; then - kill $WATCHDOG_PID 2>/dev/null || true - log "Killed registration watchdog (PID: $WATCHDOG_PID)" - else - log "Watchdog process $WATCHDOG_PID already terminated" - fi + kill $WATCHDOG_PID 2>/dev/null || true rm -f /var/run/github-runner-watchdog.pid fi - log "Starting runner" touch /var/run/github-runner-started - chmod o+x $homedir - log "Made $homedir traversable for CloudWatch agent" - mkdir -p $homedir/_diag - chmod 755 $homedir/_diag - ./run.sh + for RUNNER_DIR in $homedir/runner-*; do + [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" + done ''' # --- diff --git a/tests/test_start.py b/tests/test_start.py index da54f62..76f6548 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -34,11 +34,11 @@ def aws_params_user_data(): """User data params for AWS params tests""" return { "cloudwatch_logs_group": "", # Empty = disabled + "debug": "", # Empty = disabled "github_run_id": "16725250800", "github_run_number": "42", "github_workflow": "CI", "homedir": "/home/ec2-user", - "labels": "label", "max_instance_lifetime": "360", "repo": "omsf-eco-infra/awsinfratesting", "runner_grace_period": "61", @@ -46,9 +46,11 @@ def aws_params_user_data(): "runner_poll_interval": "11", "runner_release": "test.tar.gz", "runner_registration_timeout": "300", + "runners_per_instance": "1", + "runner_tokens": "test", # Space-delimited tokens + "runner_labels": "label", # Pipe-delimited labels "script": "echo 'Hello, World!'", "ssh_pubkey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host", - "token": "test", "userdata": "", } From f7101c6278c0a51399cc4abfa83c6ba227c97f5b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 17:09:43 -0400 Subject: [PATCH 15/69] demo-archs: {Ubuntu,Debian} x {AMD,ARM} --- .github/workflows/demo-archs.yml | 292 +++++++++++++++++++++++++------ 1 file changed, 239 insertions(+), 53 deletions(-) diff --git a/.github/workflows/demo-archs.yml b/.github/workflows/demo-archs.yml index 857a386..217c805 100644 --- a/.github/workflows/demo-archs.yml +++ b/.github/workflows/demo-archs.yml @@ -1,78 +1,264 @@ -name: Demo – x86 and ARM instances +name: Demo – OS and architecture matrix on: workflow_dispatch: + inputs: + sleep: + description: "Sleep duration in seconds for workload simulation" + required: false + type: string + default: "10" workflow_call: # Tested by `demos.yml` + inputs: + sleep: + required: false + type: string + default: "10" + permissions: id-token: write # Required for AWS OIDC authentication contents: read # Required for actions/checkout + jobs: - # Launch EC2 runners for each architecture - x86: - name: Launch x86 + # Launch EC2 runners for each OS/architecture combination + ubuntu-amd64: + name: Launch Ubuntu AMD64 uses: ./.github/workflows/runner.yml with: ec2_instance_type: t3.medium - ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) + ec2_image_id: ami-0ca5a2f40c2601df6 # Ubuntu 24.04 LTS x86_64 (us-east-1) + instance_name: "Ubuntu-AMD64/$name#$run_number" secrets: inherit - arm: - name: Launch ARM + + ubuntu-arm64: + name: Launch Ubuntu ARM64 uses: ./.github/workflows/runner.yml with: ec2_instance_type: t4g.medium ec2_image_id: ami-0aa307ed50ca3e58f # Ubuntu 24.04 LTS arm64 (us-east-1) + instance_name: "Ubuntu-ARM64/$name#$run_number" secrets: inherit - # Run jobs directly on launched instances - test-x86: - needs: x86 - if: needs.x86.outputs.id != '' - name: Test x86 - runs-on: ${{ needs.x86.outputs.id }} - steps: - - name: Architecture test on t3.medium - run: | - echo "Architecture: $(uname -m)" - echo "Processor: $(lscpu | grep 'Architecture:' | awk '{print $2}')" - echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" - test-arm: - needs: arm - if: needs.arm.outputs.id != '' - name: Test ARM - runs-on: ${{ needs.arm.outputs.id }} - steps: - - name: Architecture test on t4g.medium - run: | - echo "Architecture: $(uname -m)" - echo "Processor: $(lscpu | grep 'Architecture:' | awk '{print $2}')" - echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" + debian-amd64: + name: Launch Debian AMD64 + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: t3.large + ec2_image_id: ami-0b0012dad04fbe3d7 # Debian 13 x86_64 (us-east-1) + instance_name: "Debian-AMD64/$name#$run_number" + secrets: inherit + + debian-arm64: + name: Launch Debian ARM64 + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: t4g.large + ec2_image_id: ami-01b1eba85c1cd6a3d # Debian 13 arm64 (us-east-1) + instance_name: "Debian-ARM64/$name#$run_number" + secrets: inherit + + al2023-amd64: + name: Launch AL2023 AMD64 + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: t3.small + ec2_image_id: ami-00ca32bbc84273381 # Amazon Linux 2023 x86_64 (us-east-1) + instance_name: "AL2023-AMD64/$name#$run_number" + secrets: inherit - # Use a `matrix` to run jobs on multiple instances + al2023-arm64: + name: Launch AL2023 ARM64 + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: t4g.small + ec2_image_id: ami-0aa7db6294d00216f # Amazon Linux 2023 arm64 (us-east-1) + instance_name: "AL2023-ARM64/$name#$run_number" + secrets: inherit + + # Run test jobs on the launched instances using a matrix test-matrix: - needs: [x86, arm] - if: always() && (needs.x86.outputs.id != '' || needs.arm.outputs.id != '') - name: Test ${{ matrix.arch }} (matrix) - continue-on-error: true + needs: [ubuntu-amd64, ubuntu-arm64, debian-amd64, debian-arm64, al2023-amd64, al2023-arm64] + name: Test ${{ matrix.os }} ${{ matrix.arch }} strategy: matrix: include: - - arch: x86 - runner: ${{ needs.x86.outputs.id }} - - arch: arm - runner: ${{ needs.arm.outputs.id }} + - os: Ubuntu + arch: AMD64 + runner: ${{ needs.ubuntu-amd64.outputs.id }} + expected_arch: x86_64 + expected_os_pattern: "Ubuntu" + instance_type: t3.medium + - os: Ubuntu + arch: ARM64 + runner: ${{ needs.ubuntu-arm64.outputs.id }} + expected_arch: aarch64 + expected_os_pattern: "Ubuntu" + instance_type: t4g.medium + - os: Debian + arch: AMD64 + runner: ${{ needs.debian-amd64.outputs.id }} + expected_arch: x86_64 + expected_os_pattern: "Debian" + instance_type: t3.large + - os: Debian + arch: ARM64 + runner: ${{ needs.debian-arm64.outputs.id }} + expected_arch: aarch64 + expected_os_pattern: "Debian" + instance_type: t4g.large + - os: AL2023 + arch: AMD64 + runner: ${{ needs.al2023-amd64.outputs.id }} + expected_arch: x86_64 + expected_os_pattern: "Amazon Linux" + instance_type: t3.small + - os: AL2023 + arch: ARM64 + runner: ${{ needs.al2023-arm64.outputs.id }} + expected_arch: aarch64 + expected_os_pattern: "Amazon Linux" + instance_type: t4g.small fail-fast: false runs-on: ${{ matrix.runner }} steps: - - name: Matrix architecture test - run: | - echo "Running on ${{ matrix.arch }}" - echo "Architecture: $(uname -m)" - echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" - - # Verify expected architecture - if [[ "${{ matrix.arch }}" == "x86" ]]; then - echo "Verifying x86_64 architecture..." - [[ "$(uname -m)" == "x86_64" ]] && echo "✓ Confirmed x86_64" || exit 1 - elif [[ "${{ matrix.arch }}" == "arm" ]]; then - echo "Verifying ARM architecture..." - [[ "$(uname -m)" == "aarch64" ]] && echo "✓ Confirmed aarch64" || exit 1 - fi + - name: System information + run: | + echo "=== System Information ===" + echo "OS: ${{ matrix.os }}" + echo "Architecture: ${{ matrix.arch }}" + echo "Expected instance type: ${{ matrix.instance_type }}" + echo "" + echo "=== Actual System Details ===" + echo "Hostname: $(hostname)" + echo "Kernel: $(uname -r)" + echo "Architecture: $(uname -m)" + echo "OS Release:" + cat /etc/os-release | head -5 + echo "" + echo "=== AWS Instance Details ===" + echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" + echo "Region: $(curl -s http://169.254.169.254/latest/meta-data/placement/region)" + echo "AZ: $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)" + + - name: Verify OS and architecture + run: | + echo "=== Verification ===" + ACTUAL_ARCH=$(uname -m) + OS_NAME=$(grep '^NAME=' /etc/os-release | cut -d'"' -f2) + ACTUAL_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) + + # Verify architecture + if [[ "$ACTUAL_ARCH" == "${{ matrix.expected_arch }}" ]]; then + echo "✓ Architecture verified: $ACTUAL_ARCH matches expected ${{ matrix.expected_arch }}" + else + echo "✗ Architecture mismatch: got $ACTUAL_ARCH, expected ${{ matrix.expected_arch }}" + exit 1 + fi + + # Verify OS + if echo "$OS_NAME" | grep -q "${{ matrix.expected_os_pattern }}"; then + echo "✓ OS verified: $OS_NAME matches expected pattern '${{ matrix.expected_os_pattern }}'" + else + echo "✗ OS mismatch: $OS_NAME does not match expected pattern '${{ matrix.expected_os_pattern }}'" + exit 1 + fi + + # Verify instance type + if [[ "$ACTUAL_INSTANCE_TYPE" == "${{ matrix.instance_type }}" ]]; then + echo "✓ Instance type verified: $ACTUAL_INSTANCE_TYPE" + else + echo "✗ Instance type mismatch: got $ACTUAL_INSTANCE_TYPE, expected ${{ matrix.instance_type }}" + exit 1 + fi + + - name: Performance characteristics + run: | + echo "=== Performance Characteristics ===" + echo "CPU cores: $(nproc)" + echo "CPU model: $(lscpu | grep 'Model name:' | cut -d: -f2 | xargs)" + echo "Memory: $(free -h | grep Mem | awk '{print $2}')" + echo "Disk: $(df -h / | tail -1 | awk '{print $2}')" + echo "" + echo "CPU details:" + lscpu | grep -E "Architecture:|CPU\(s\):|Thread\(s\) per core:|Core\(s\) per socket:|Socket\(s\):" + + - name: Simulate workload + run: | + DURATION=${{ inputs.sleep }} + echo "=== Workload Simulation ===" + echo "Starting workload at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" + echo "Running on ${{ matrix.os }} ${{ matrix.arch }} for ${DURATION} seconds..." + + # Architecture-specific workload + if [[ "${{ matrix.arch }}" == "ARM64" ]]; then + echo "Running ARM-optimized workload simulation..." + # Simulate ARM-specific operations + echo "ARM NEON/SVE capabilities check:" + grep -E "Features|flags" /proc/cpuinfo | head -2 || true + else + echo "Running x86-optimized workload simulation..." + # Simulate x86-specific operations + echo "x86 instruction set extensions:" + grep -E "flags" /proc/cpuinfo | head -1 | grep -o -E "sse|avx|avx2|avx512" | sort -u || true + fi + + sleep $DURATION + echo "Workload complete at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" + + - name: Package manager test + run: | + echo "=== Package Manager Test ===" + if [[ "${{ matrix.os }}" == "Ubuntu" ]]; then + echo "Ubuntu uses APT package manager" + echo "APT version: $(apt-get --version | head -1)" + echo "Distribution: $(lsb_release -ds 2>/dev/null || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + elif [[ "${{ matrix.os }}" == "Debian" ]]; then + echo "Debian uses APT package manager" + echo "APT version: $(apt-get --version | head -1)" + echo "Distribution: $(cat /etc/debian_version)" + elif [[ "${{ matrix.os }}" == "AL2023" ]]; then + echo "Amazon Linux 2023 uses DNF/YUM package manager" + echo "DNF version: $(dnf --version | head -1)" + echo "Distribution: $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + fi + + # Show architecture-specific packages available + echo "" + echo "Architecture-specific package availability:" + if [[ "${{ matrix.arch }}" == "ARM64" ]]; then + echo "Checking for ARM64-specific packages..." + if [[ "${{ matrix.os }}" == "AL2023" ]]; then + rpm --eval '%{_arch}' + dnf repolist | head -5 + else + dpkg --print-architecture + dpkg --print-foreign-architectures || echo "No foreign architectures configured" + fi + else + echo "Checking for AMD64-specific packages..." + if [[ "${{ matrix.os }}" == "AL2023" ]]; then + rpm --eval '%{_arch}' + dnf repolist | head -5 + else + dpkg --print-architecture + dpkg --print-foreign-architectures || echo "No foreign architectures configured" + fi + fi + + - name: Summary + run: | + echo "=== Test Summary ===" + echo "✓ Successfully tested ${{ matrix.os }} on ${{ matrix.arch }} architecture" + echo "✓ Instance type: ${{ matrix.instance_type }}" + echo "✓ All verifications passed" + echo "" + echo "Matrix configuration demonstrated:" + echo "- OS diversity: Ubuntu 24.04, Debian 13, and Amazon Linux 2023" + echo "- Architecture diversity: AMD64 (x86_64) and ARM64 (aarch64)" + echo "- Instance type variety: t3.small, t3.medium, t3.large, t4g.small, t4g.medium, t4g.large" + echo "" + echo "This workflow shows ec2-gha's ability to:" + echo "1. Launch runners on different OS distributions" + echo "2. Support both x86 and ARM architectures" + echo "3. Use appropriate instance types for each architecture" + echo "4. Run parallel jobs across diverse infrastructure" \ No newline at end of file From 7a3a563d98d64ec0f36037e231decfd79b9eff19 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 20:19:28 -0400 Subject: [PATCH 16/69] typing lints, `environ` --- scripts/instance-runtime.py | 17 ++++++++--------- src/ec2_gha/__main__.py | 9 ++++----- src/ec2_gha/start.py | 22 +++++++++++----------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/scripts/instance-runtime.py b/scripts/instance-runtime.py index c0e7428..0ca0517 100755 --- a/scripts/instance-runtime.py +++ b/scripts/instance-runtime.py @@ -14,9 +14,8 @@ import subprocess import sys from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from functools import partial -from typing import Dict, List, Optional from dateutil import parser as date_parser @@ -34,7 +33,7 @@ err = partial(print, file=sys.stderr) -def run_command(cmd: List[str]) -> Optional[str]: +def run_command(cmd: list[str]) -> str | None: """Run a command and return output, or None on error.""" try: result = subprocess.run(cmd, capture_output=True, text=True, check=True) @@ -49,7 +48,7 @@ def run_command(cmd: List[str]) -> Optional[str]: -def get_log_streams(instance_id: str, log_group: str = None) -> List[Dict]: +def get_log_streams(instance_id: str, log_group: str = None) -> list[dict]: if log_group is None: log_group = DEFAULT_CLOUDWATCH_LOG_GROUP """Get CloudWatch log streams for an instance.""" @@ -69,7 +68,7 @@ def get_log_streams(instance_id: str, log_group: str = None) -> List[Dict]: return [] -def get_log_events(log_group: str, log_stream: str, limit: int = 100, start_from_head: bool = False) -> List[Dict]: +def get_log_events(log_group: str, log_stream: str, limit: int = 100, start_from_head: bool = False) -> list[dict]: """Get events from a CloudWatch log stream.""" cmd = [ "aws", "logs", "get-log-events", @@ -91,7 +90,7 @@ def get_log_events(log_group: str, log_stream: str, limit: int = 100, start_from return [] -def parse_timestamp(ts_str: str) -> Optional[datetime]: +def parse_timestamp(ts_str: str) -> datetime | None: """Parse various timestamp formats and ensure timezone is set.""" try: dt = date_parser.parse(ts_str) @@ -103,7 +102,7 @@ def parse_timestamp(ts_str: str) -> Optional[datetime]: return None -def extract_timestamp_from_log(message: str) -> Optional[datetime]: +def extract_timestamp_from_log(message: str) -> datetime | None: """Extract timestamp from log message.""" # Pattern: [Thu Aug 14 00:29:25 UTC 2025] match = re.search(r'\[([^]]+UTC \d{4})\]', message) @@ -118,7 +117,7 @@ def extract_timestamp_from_log(message: str) -> Optional[datetime]: return None -def analyze_instance(instance_id: str, log_group: str = None) -> Dict: +def analyze_instance(instance_id: str, log_group: str = None) -> dict: if log_group is None: log_group = DEFAULT_CLOUDWATCH_LOG_GROUP """Analyze runtime and job execution for an instance.""" @@ -307,7 +306,7 @@ def analyze_instance(instance_id: str, log_group: str = None) -> Dict: return result -def get_instances_from_github_url(url: str) -> List[str]: +def get_instances_from_github_url(url: str) -> list[str]: """Extract instance IDs from a GitHub Actions URL.""" # Parse the URL match = re.match(r'https://github\.com/([^/]+)/([^/]+)/actions/runs/(\d+)(?:/job/(\d+))?', url) diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index a7b506a..f0b5b18 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -12,22 +12,21 @@ from gha_runner.gh import GitHubInstance from gha_runner.clouddeployment import DeployInstance from gha_runner.helper.input import EnvVarBuilder, check_required -import os from os import environ def main(): - env = dict(os.environ) + env = dict(environ) required = ["GH_PAT", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] # Check that everything exists check_required(env, required) # Timeout for waiting for runner to register with GitHub - timeout_str = os.environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "").strip() + timeout_str = environ.get("INPUT_RUNNER_REGISTRATION_TIMEOUT", "").strip() timeout = int(timeout_str) if timeout_str else int(RUNNER_REGISTRATION_TIMEOUT) - token = os.environ["GH_PAT"] + token = environ["GH_PAT"] # Make a copy of environment variables for immutability - env = dict(os.environ) + env = dict(environ) builder = ( EnvVarBuilder(env) diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 17b43a1..3d3804c 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -135,16 +135,16 @@ def _build_aws_params(self, user_data_params: dict, idx: int = None) -> dict: template_vars = {} # Get repository name (just the basename) - if os.environ.get("GITHUB_REPOSITORY"): - template_vars["repo"] = os.environ["GITHUB_REPOSITORY"].split("/")[-1] + if environ.get("GITHUB_REPOSITORY"): + template_vars["repo"] = environ["GITHUB_REPOSITORY"].split("/")[-1] else: template_vars["repo"] = "unknown" # Get workflow full name (e.g., "Test pip install") - template_vars["workflow"] = os.environ.get("GITHUB_WORKFLOW", "unknown") + template_vars["workflow"] = environ.get("GITHUB_WORKFLOW", "unknown") # Get workflow filename stem and ref from GITHUB_WORKFLOW_REF - workflow_ref = os.environ.get("GITHUB_WORKFLOW_REF", "") + workflow_ref = environ.get("GITHUB_WORKFLOW_REF", "") if workflow_ref: import re # Extract filename and ref from path like "owner/repo/.github/workflows/test.yml@ref" @@ -168,7 +168,7 @@ def _build_aws_params(self, user_data_params: dict, idx: int = None) -> dict: template_vars["ref"] = "unknown" # Get run number - template_vars["run_number"] = os.environ.get("GITHUB_RUN_NUMBER", "unknown") + template_vars["run_number"] = environ.get("GITHUB_RUN_NUMBER", "unknown") # Add instance index if provided (for multi-instance launches) if idx is not None: @@ -182,16 +182,16 @@ def _build_aws_params(self, user_data_params: dict, idx: int = None) -> dict: default_tags.append({"Key": "Name", "Value": name_value}) # Add repository tag if available - if "Repository" not in existing_keys and os.environ.get("GITHUB_REPOSITORY"): - default_tags.append({"Key": "Repository", "Value": os.environ["GITHUB_REPOSITORY"]}) + if "Repository" not in existing_keys and environ.get("GITHUB_REPOSITORY"): + default_tags.append({"Key": "Repository", "Value": environ["GITHUB_REPOSITORY"]}) # Add workflow tag if available - if "Workflow" not in existing_keys and os.environ.get("GITHUB_WORKFLOW"): - default_tags.append({"Key": "Workflow", "Value": os.environ["GITHUB_WORKFLOW"]}) + if "Workflow" not in existing_keys and environ.get("GITHUB_WORKFLOW"): + default_tags.append({"Key": "Workflow", "Value": environ["GITHUB_WORKFLOW"]}) # Add run URL tag if available - if "URL" not in existing_keys and os.environ.get("GITHUB_SERVER_URL") and os.environ.get("GITHUB_REPOSITORY") and os.environ.get("GITHUB_RUN_ID"): - gha_url = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{os.environ['GITHUB_RUN_ID']}" + if "URL" not in existing_keys and environ.get("GITHUB_SERVER_URL") and environ.get("GITHUB_REPOSITORY") and environ.get("GITHUB_RUN_ID"): + gha_url = f"{environ['GITHUB_SERVER_URL']}/{environ['GITHUB_REPOSITORY']}/actions/runs/{environ['GITHUB_RUN_ID']}" default_tags.append({"Key": "URL", "Value": gha_url}) # Combine user tags with default tags From ea6bb7d63ed03a63da479ac996410d3102d81414 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 21:30:26 -0400 Subject: [PATCH 17/69] sleep 10m in debug mode only --- src/ec2_gha/templates/user-script.sh.templ | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index f78eca4..6449aab 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -73,13 +73,20 @@ terminate_instance() { flush_cloudwatch_logs - # Give time for debugging - log "Sleeping for 300 seconds before shutdown to allow debugging..." - log "SSH into instance with: ssh ubuntu@$$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 300 + # Give time for debugging only in debug mode + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + # Detect the SSH user from the home directory + local ssh_user=$$(basename "$$homedir" 2>/dev/null || echo "ec2-user") + local public_ip=$$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) + log "SSH into instance with: ssh $${ssh_user}@$${public_ip}" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down immediately (debug mode not enabled)" + fi - log "Shutting down after debug delay" shutdown -h now exit 1 } From 1f877951587b632e2aeb77f59f363ad248db4ae6 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 22:02:48 -0400 Subject: [PATCH 18/69] CLAUDE.md --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index a32201f..f774380 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ ## Common Development Commands - Don't explicitly set the `AWS_PROFILE` (e.g. to `oa-ci-dev`) in your commands; assume it's set for you out of band, verify if you need. -- Instance userdata (rendered form of `src/ec2_gha/templates/user-script.sh.templ`) has to stay under 16KiB. +- Instance userdata (rendered form of `src/ec2_gha/templates/user-script.sh.templ`) has to stay under 16KiB. We remove comments while "rendering", so the `templ` itself may be a bit over the limit. ### Testing ```bash From b80f9598dddb47ce22393893588ff449e5d22fab Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 22:19:41 -0400 Subject: [PATCH 19/69] `demo-dbg-minimal.yml` --- .github/workflows/demo-archs.yml | 6 +- .github/workflows/demo-dbg-minimal.yml | 119 +++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/demo-dbg-minimal.yml diff --git a/.github/workflows/demo-archs.yml b/.github/workflows/demo-archs.yml index 217c805..70c972b 100644 --- a/.github/workflows/demo-archs.yml +++ b/.github/workflows/demo-archs.yml @@ -43,7 +43,7 @@ jobs: uses: ./.github/workflows/runner.yml with: ec2_instance_type: t3.large - ec2_image_id: ami-0b0012dad04fbe3d7 # Debian 13 x86_64 (us-east-1) + ec2_image_id: ami-05b50089e01b13194 # Debian 12 x86_64 (us-east-1) instance_name: "Debian-AMD64/$name#$run_number" secrets: inherit @@ -52,7 +52,7 @@ jobs: uses: ./.github/workflows/runner.yml with: ec2_instance_type: t4g.large - ec2_image_id: ami-01b1eba85c1cd6a3d # Debian 13 arm64 (us-east-1) + ec2_image_id: ami-0505441d7e1514742 # Debian 12 arm64 (us-east-1) instance_name: "Debian-ARM64/$name#$run_number" secrets: inherit @@ -261,4 +261,4 @@ jobs: echo "1. Launch runners on different OS distributions" echo "2. Support both x86 and ARM architectures" echo "3. Use appropriate instance types for each architecture" - echo "4. Run parallel jobs across diverse infrastructure" \ No newline at end of file + echo "4. Run parallel jobs across diverse infrastructure" diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml new file mode 100644 index 0000000..ccfac89 --- /dev/null +++ b/.github/workflows/demo-dbg-minimal.yml @@ -0,0 +1,119 @@ +name: Demo – configurable instance for debugging +on: + workflow_dispatch: + inputs: + type: + description: "EC2 instance type (e.g., t3.medium, t3.large)" + required: false + type: string + default: "t3.large" + ami: + description: "AMI ID to use" + required: false + type: string + default: "ami-0e86e20dae9224db8" # Ubuntu 24.04 LTS x86_64 (us-east-1) + registration_timeout: + description: "Max seconds to wait for runner registration (default: 360)" + required: false + type: string + default: "360" + initial_grace_period: + description: "Grace period if no job starts (default: 180)" + required: false + type: string + default: "180" + grace_period: + description: "Grace period before termination after job (default: 60)" + required: false + type: string + default: "60" + max_instance_lifetime: + description: "Max instance lifetime in minutes (default: 360)" + required: false + type: string + default: "360" + sleep: + description: "Sleep duration in seconds" + required: false + type: string + default: "600" + debug: + description: "Enable debug mode (extends termination delay to 600s)" + required: false + type: boolean + default: true + +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout + +jobs: + launch: + name: Launch Debug Instance + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: ${{ inputs.type }} + ec2_image_id: ${{ inputs.ami }} + debug: ${{ inputs.debug }} + instance_name: "debug/$name#$run_number" + runner_registration_timeout: ${{ inputs.registration_timeout }} + runner_grace_period: ${{ inputs.grace_period }} + runner_initial_grace_period: ${{ inputs.initial_grace_period }} + max_instance_lifetime: ${{ inputs.max_instance_lifetime }} + secrets: inherit + + test: + needs: launch + name: Test Job + runs-on: ${{ needs.launch.outputs.id }} + steps: + - name: Instance Info + run: | + echo "=== Instance Information ===" + echo "Hostname: $(hostname)" + echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" + echo "Public IP: $(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + echo "Region: $(curl -s http://169.254.169.254/latest/meta-data/placement/region)" + echo "AMI ID: $(curl -s http://169.254.169.254/latest/meta-data/ami-id)" + echo "" + echo "=== System Details ===" + echo "OS: $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + echo "Kernel: $(uname -r)" + echo "Architecture: $(uname -m)" + echo "CPU cores: $(nproc)" + echo "Memory: $(free -h | grep Mem | awk '{print $2}')" + echo "User: $(whoami)" + echo "Home: $HOME" + + - name: Sleep Test + run: | + DURATION=${{ inputs.sleep }} + echo "Starting sleep at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" + echo "Sleeping for ${DURATION} seconds..." + sleep $DURATION + echo "Finished at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" + + - name: Debug Info + run: | + echo "=== Runner Environment ===" + echo "Runner home: ${RUNNER_HOME:-not set}" + echo "Runner workspace: ${GITHUB_WORKSPACE:-not set}" + echo "Runner temp: ${RUNNER_TEMP:-not set}" + echo "" + echo "=== Timeout Settings ===" + echo "Debug mode: ${{ inputs.debug }}" + echo "Registration timeout: ${{ inputs.registration_timeout }}s" + echo "Grace period: ${{ inputs.grace_period }}s" + echo "Initial grace period: ${{ inputs.initial_grace_period }}s" + echo "Max lifetime: ${{ inputs.max_instance_lifetime }} minutes" + echo "" + echo "=== Important Logs ===" + echo "Setup log: /var/log/runner-setup.log" + echo "Debug log: /var/log/runner-debug.log" + if [ -f /tmp/runner-0-config.log ]; then + echo "Config log exists: /tmp/runner-0-config.log" + echo "Last 5 lines of config log:" + tail -5 /tmp/runner-0-config.log + fi + From 4189ba879b0848bfb8fbd2972b70be3f4310c286 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 23:59:30 -0400 Subject: [PATCH 20/69] CW: `/tmp/runner-*-config.log` --- src/ec2_gha/templates/user-script.sh.templ | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 6449aab..f8c4de4 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -226,6 +226,7 @@ if [ "$cloudwatch_logs_group" != "" ]; then { "file_path": "/tmp/job-started-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, { "file_path": "/tmp/termination-check.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, + { "file_path": "/tmp/runner-*-config.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-config", "timezone": "UTC" }, { "file_path": "$homedir/_diag/Runner_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, { "file_path": "$homedir/_diag/Worker_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } ] From a096a5b67a24fb880166500a223b328ce693982d Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 23:59:36 -0400 Subject: [PATCH 21/69] debug mode shutdown --- src/ec2_gha/templates/user-script.sh.templ | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index f8c4de4..d6cfa13 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -357,7 +357,14 @@ if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then fi done flush_cloudwatch_logs - log "Shutting down" + # In debug mode, give time for debugging before shutdown + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping 600 seconds before shutdown..." + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down" + fi sudo shutdown -h now else [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" From 05e1e73ebbd844c4eb074b0bb34ab61a99d5730f Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 19 Aug 2025 23:59:55 -0400 Subject: [PATCH 22/69] `./bin/installdependencies.sh` for AL2023 / Debian 13 --- src/ec2_gha/templates/user-script.sh.templ | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index d6cfa13..7c05f98 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -439,6 +439,12 @@ configure_runner() { # Extract runner tar -xzf /tmp/runner.tar.gz + # Install dependencies if needed (for AL2023, Debian 13, etc.) + if [ -f ./bin/installdependencies.sh ]; then + log "Installing runner dependencies..." + sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + fi + # Save token for deregistration echo "$$token" > .runner-token From 5ba8ef1a5a6034147e5a1744f323fa4a3cd33ef3 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 00:24:11 -0400 Subject: [PATCH 23/69] `shared-functions.sh.templ` --- CLAUDE.md | 4 +- src/ec2_gha/start.py | 13 + .../templates/shared-functions.sh.templ | 157 ++ src/ec2_gha/templates/user-script.sh.templ | 408 ++--- tests/__snapshots__/test_start.ambr | 1480 ++++++++--------- 5 files changed, 972 insertions(+), 1090 deletions(-) create mode 100644 src/ec2_gha/templates/shared-functions.sh.templ diff --git a/CLAUDE.md b/CLAUDE.md index f774380..d191249 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ # Install test dependencies pip install '.[test]' -# Run tests matching a pattern +# Run tests matching a pattern. You don't need to do this very often though. cd tests/ && pytest -v -m 'not slow' # Update `syrupy` "snapshots", run tests to verify they pass with (possibly-updated) snapshot values. Just a wrapper for: @@ -20,7 +20,7 @@ cd tests/ && pytest -v -m 'not slow' # pytest --snapshot-update -m 'not slow' # pytest -vvv -m 'not slow' . # ``` -# Can be used in conjunction with `git rebase -x`. +# Can be used in conjunction with `git rebase -x`. I'll mostly do this manually when cleaning up commits. scripts/update-snapshots.sh ``` diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 3d3804c..f525c8d 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -227,6 +227,19 @@ def _build_user_data(self, **kwargs) -> str: kwargs['log_prefix_job_started'] = LOG_PREFIX_JOB_STARTED kwargs['log_prefix_job_completed'] = LOG_PREFIX_JOB_COMPLETED + # Load shared functions template - don't render it, just include as-is + shared_functions_template = importlib.resources.files("ec2_gha").joinpath("templates/shared-functions.sh.templ") + with shared_functions_template.open() as f: + shared_functions_content = f.read() + + # Strip the shebang line from shared functions since it will be embedded + shared_functions_lines = shared_functions_content.split('\n') + if shared_functions_lines[0].startswith('#!'): + shared_functions_content = '\n'.join(shared_functions_lines[1:]) + + # Add shared functions as-is to the main template kwargs + kwargs['shared_functions'] = shared_functions_content + template = importlib.resources.files("ec2_gha").joinpath("templates/user-script.sh.templ") with template.open() as f: template_content = f.read() diff --git a/src/ec2_gha/templates/shared-functions.sh.templ b/src/ec2_gha/templates/shared-functions.sh.templ new file mode 100644 index 0000000..101627f --- /dev/null +++ b/src/ec2_gha/templates/shared-functions.sh.templ @@ -0,0 +1,157 @@ +#!/bin/bash +# Shared functions for runner scripts +# These functions are used by multiple scripts throughout the runner lifecycle + +# Logging functions +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } +log_error() { log "ERROR: $1" >&2; } + +# Wait for dpkg lock to be released (for Debian/Ubuntu systems) +wait_for_dpkg_lock() { + local timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done +} + +# Function to flush CloudWatch logs before shutdown +flush_cloudwatch_logs() { + log "Stopping CloudWatch agent to flush logs" + if systemctl is-active --quiet amazon-cloudwatch-agent; then + systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true + fi +} + +# Get EC2 instance metadata (IMDSv2 compatible) +get_metadata() { + local path="$1" + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + if [ -n "$token" ]; then + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + else + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + fi + return 0 # Always return success to avoid set -e issues +} + +# Function to deregister all runners +deregister_all_runners() { + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in $RUNNER_DIR" + cd "$RUNNER_DIR" + pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "$RUNNER_DIR/.runner-token" ]; then + TOKEN=$(cat "$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 + log "Deregistration exit: $?" + fi + fi + done +} + +# Function to handle debug mode sleep and shutdown +debug_sleep_and_shutdown() { + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + # Detect the SSH user from the home directory + local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) + log "SSH into instance with: ssh ${ssh_user}@${public_ip}" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down immediately (debug mode not enabled)" + fi + shutdown -h now +} + +# Function to handle fatal errors and terminate the instance +terminate_instance() { + local reason="$1" + local instance_id=$(get_metadata "instance-id") + + # Log error prominently + echo "========================================" | tee -a /var/log/runner-setup.log + log "FATAL ERROR DETECTED" + log "Reason: $reason" + log "Instance: $instance_id" + log "Script location: $(pwd)" + log "User: $(whoami)" + log "Debug trace available in: /var/log/runner-debug.log" + echo "========================================" | tee -a /var/log/runner-setup.log + + # Try to remove runner if it was partially configured + if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then + cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true + fi + + flush_cloudwatch_logs + debug_sleep_and_shutdown + exit 1 +} + +# Function to configure a single GitHub Actions runner +configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 + + log "Configuring runner $idx..." + + # Create runner directory and extract runner binary + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + tar -xzf /tmp/runner.tar.gz + + # Install dependencies if needed (for AL2023, Debian 13, etc.) + if [ -f ./bin/installdependencies.sh ]; then + log "Installing runner dependencies..." + sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + fi + + # Save token for deregistration + echo "$token" > .runner-token + + # Create env file with runner hooks + cat > .env << EOF +ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh +ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh +RUNNER_HOME=$runner_dir +RUNNER_INDEX=$idx +RUNNER_GRACE_PERIOD=$runner_grace_period +RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period +EOF + + # Configure runner with GitHub + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi + + # Start runner in background + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" + + return 0 +} \ No newline at end of file diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 7c05f98..fbe07f5 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -4,97 +4,63 @@ set -e # Enable debug tracing to a file for troubleshooting exec 2> >(tee -a /var/log/runner-debug.log >&2) +# Set debug variable for use in shared functions +debug="$debug" + # Conditionally enable debug mode -if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - echo "[DEBUG] Debug mode enabled - set -x active" >&2 - set -x -fi +[ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ] && set -x -log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } -log_error() { log "ERROR: $$1" >&2; } +# Determine home directory early since it's needed by shared functions +homedir="$homedir" +if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$$user" &>/dev/null; then + homedir="/home/$$user" + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $$homedir" | tee -a /var/log/runner-setup.log + break + fi + done -# Function to flush CloudWatch logs before shutdown -flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true + # Fallback if no standard user found + if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then + homedir=$$(getent passwd | awk -F: '$$3 >= 1000 && $$3 < 65534 && $$6 ~ /^\\/home\\// {print $$6}' | while read dir; do + if [ -d "$$dir" ]; then + echo "$$dir" + break + fi + done) + if [ -z "$$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $$homedir" | tee -a /var/log/runner-setup.log + else + owner=$$(stat -c "%U" "$$homedir" 2>/dev/null || stat -f "%Su" "$$homedir" 2>/dev/null) + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $$homedir (owner: $$owner)" | tee -a /var/log/runner-setup.log + fi fi -} +else + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $$homedir" | tee -a /var/log/runner-setup.log +fi +export homedir -# Create common functions file -cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' -#!/bin/bash -log() { echo "[$$(date '+%Y-%m-%d %H:%M:%S')] $$1" | tee -a /var/log/runner-setup.log; } -flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true - fi -} -EOCF - -# Get metadata (IMDSv2 compatible) -get_metadata() { - local path="$$1" - local token=$$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) - if [ -n "$$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $$token" "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$$path" 2>/dev/null || echo "unknown" - fi - return 0 # Always return success to avoid set -e issues -} +# Write shared functions that will be used by multiple scripts +cat > /usr/local/bin/runner-common.sh << EOSF +# Auto-generated shared functions and variables +# Set homedir for scripts that source this file +homedir="$$homedir" +debug="$$debug" +export homedir debug + +$shared_functions +EOSF +chmod +x /usr/local/bin/runner-common.sh +source /usr/local/bin/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR - -terminate_instance() { - local reason="$$1" - local instance_id=$$(get_metadata "instance-id") - - # Log error prominently - echo "========================================" | tee -a /var/log/runner-setup.log - log "FATAL ERROR DETECTED" - log "Reason: $$reason" - log "Instance: $$instance_id" - log "Script location: $$(pwd)" - log "User: $$(whoami)" - log "Debug trace available in: /var/log/runner-debug.log" - echo "========================================" | tee -a /var/log/runner-setup.log - - # Try to remove runner if it was partially configured - if [ -f "$$homedir/config.sh" ] && [ -n "$${RUNNER_TOKEN:-}" ]; then - cd "$$homedir" && ./config.sh remove --token "$${RUNNER_TOKEN}" || true - fi - - flush_cloudwatch_logs - - # Give time for debugging only in debug mode - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." - # Detect the SSH user from the home directory - local ssh_user=$$(basename "$$homedir" 2>/dev/null || echo "ec2-user") - local public_ip=$$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) - log "SSH into instance with: ssh $${ssh_user}@$${public_ip}" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 600 - log "Debug period ended, shutting down" - else - log "Shutting down immediately (debug mode not enabled)" - fi - - shutdown -h now - exit 1 -} - -# Trap errors and ensure termination on failure trap 'terminate_instance "Setup script failed with error on line $$LINENO"' ERR -# Set up registration timeout failsafe +# Set up registration timeout failsafe - terminate if runner doesn't register in time REGISTRATION_TIMEOUT="$runner_registration_timeout" # Validate timeout is a number, default to 300 if not if ! [[ "$$REGISTRATION_TIMEOUT" =~ ^[0-9]+$$ ]]; then @@ -114,90 +80,36 @@ logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" ) & REGISTRATION_WATCHDOG_PID=$$! log "Watchdog PID: $$REGISTRATION_WATCHDOG_PID" -# Save watchdog PID to file so it survives exec redirect echo $$REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid +# Run any custom user data script provided by the user $userdata -# Initialize homedir from template variable -homedir="$homedir" - -# Determine home directory if not specified or set to AUTO -if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - # Check for common cloud-init created users - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$$user" &>/dev/null; then - homedir="/home/$$user" - log "Auto-detected: $$homedir" - break - fi - done - - # Fallback if no standard user found - if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then - # Get the first non-root user's home directory that actually exists - homedir=$$(getent passwd | awk -F: '$$3 >= 1000 && $$3 < 65534 && $$6 ~ /^\/home\// {print $$6}' | while read dir; do - if [ -d "$$dir" ]; then - echo "$$dir" - break - fi - done) - - if [ -z "$$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $$homedir" - else - # Use stat to get the actual owner of the directory - owner=$$(stat -c "%U" "$$homedir" 2>/dev/null || stat -f "%Su" "$$homedir" 2>/dev/null) - log "Detected: $$homedir ($$owner)" - fi - fi -else - log "Using: $$homedir" -fi - -# Redirect runner setup logs to a file for CloudWatch exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" -# Fetch instance metadata +# Fetch instance metadata for labeling and logging INSTANCE_TYPE=$$(get_metadata "instance-type") INSTANCE_ID=$$(get_metadata "instance-id") REGION=$$(get_metadata "placement/region") AZ=$$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=$${INSTANCE_TYPE} ID=$${INSTANCE_ID} Region=$${REGION} AZ=$${AZ}" -# Set up maximum lifetime timeout - do this early to ensure cleanup +# Set up maximum lifetime timeout - instance will terminate after this time regardless of job status MAX_LIFETIME_MINUTES=$max_instance_lifetime log "Setting up maximum lifetime timeout: $${MAX_LIFETIME_MINUTES} minutes" nohup bash -c "sleep $${MAX_LIFETIME_MINUTES}m && echo '[$$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & -# Configure CloudWatch Logs if enabled +# Configure CloudWatch Logs if a log group is specified if [ "$cloudwatch_logs_group" != "" ]; then log "Installing CloudWatch agent" # Use a subshell to prevent CloudWatch failures from stopping the entire script ( - # Detect package manager and install CloudWatch agent if command -v dpkg >/dev/null 2>&1; then # Debian/Ubuntu log "Detected dpkg-based system" - - # Wait for dpkg lock to be released (up to 2 minutes) - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $$timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($$timeout seconds remaining)" - sleep 5 - timeout=$$((timeout - 5)) - done - + wait_for_dpkg_lock wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb @@ -210,78 +122,51 @@ if [ "$cloudwatch_logs_group" != "" ]; then else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi - - # Configure CloudWatch agent cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF -{ - "agent": { - "run_as_user": "cwagent" - }, - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { "file_path": "/var/log/runner-setup.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, - { "file_path": "/var/log/runner-debug.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, - { "file_path": "/tmp/job-started-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, - { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, - { "file_path": "/tmp/termination-check.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, - { "file_path": "/tmp/runner-*-config.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-config", "timezone": "UTC" }, - { "file_path": "$homedir/_diag/Runner_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, - { "file_path": "$homedir/_diag/Worker_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } - ] - } - } - } -} +{"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ +{"file_path":"/var/log/runner-setup.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, +{"file_path":"/var/log/runner-debug.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, +{"file_path":"/tmp/job-started-hook.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, +{"file_path":"/tmp/job-completed-hook.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, +{"file_path":"/tmp/termination-check.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, +{"file_path":"/tmp/runner-*-config.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, +{"file_path":"$homedir/_diag/Runner_**.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, +{"file_path":"$homedir/_diag/Worker_**.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} +]}}}} EOF - - # Start CloudWatch agent - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s - + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s log "CloudWatch agent started" ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi -# Configure SSH access if public key provided +# Configure SSH access if public key provided (useful for debugging) if [ -n "$ssh_pubkey" ]; then log "Configuring SSH access" - # Determine the default user based on the home directory owner DEFAULT_USER=$$(stat -c "%U" "$$homedir" 2>/dev/null || echo "root") - - # Create .ssh directory if it doesn't exist - mkdir -p "$homedir/.ssh" - chmod 700 "$homedir/.ssh" - - # Add the public key to authorized_keys + mkdir -p "$$homedir/.ssh" + chmod 700 "$$homedir/.ssh" echo "$ssh_pubkey" >> "$$homedir/.ssh/authorized_keys" chmod 600 "$$homedir/.ssh/authorized_keys" - - # Set proper ownership if [ "$$DEFAULT_USER" != "root" ]; then chown -R "$$DEFAULT_USER:$$DEFAULT_USER" "$$homedir/.ssh" fi - log "SSH key added for user $$DEFAULT_USER" fi log "Working directory: $$homedir" cd "$$homedir" +# Run any pre-runner script provided by the user echo "$script" > pre-runner-script.sh log "Running pre-runner script" source pre-runner-script.sh export RUNNER_ALLOW_RUNASROOT=1 -# Runner configurations are always provided as JSON from Python +# Number of runners to configure on this instance RUNNERS_PER_INSTANCE=$runners_per_instance -# Download runner binary once +# Download GitHub Actions runner binary ARCH=$$(uname -m) if [ "$$ARCH" = "aarch64" ] || [ "$$ARCH" = "arm64" ]; then RUNNER_URL=$$(echo "$runner_release" | sed 's/x64/arm64/g') @@ -291,7 +176,6 @@ else log "x64 detected, using: $$RUNNER_URL" fi -# Check for curl or wget if command -v curl >/dev/null 2>&1; then curl -L $$RUNNER_URL -o /tmp/runner.tar.gz elif command -v wget >/dev/null 2>&1; then @@ -301,7 +185,9 @@ else terminate_instance "No download tool available" fi log "Downloaded runner binary" -# Create shared job tracking scripts (used by all runners) + +# Create job tracking scripts - these are called by GitHub runner hooks +# job-started-hook.sh is called when a job starts cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 @@ -309,10 +195,11 @@ V="/var/run/github-runner" RUNNER_IDX="$${RUNNER_INDEX:-0}" echo "[$$(date)] Runner-$$RUNNER_IDX: ${log_prefix_job_started} $${GITHUB_JOB}" mkdir -p $$V-jobs -echo '{"status":"running","runner":"'$$RUNNER_IDX'"}' > $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job +echo '{\"status\":\"running\",\"runner\":\"'$$RUNNER_IDX'\"}' > $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job touch $$V-last-activity $$V-has-run-job EOFS +# job-completed-hook.sh is called when a job completes cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 @@ -323,55 +210,32 @@ rm -f $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job touch $$V-last-activity EOFC +# check-runner-termination.sh is called periodically to check if the instance should terminate cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 -source /usr/local/bin/runner-common-functions.sh +source /usr/local/bin/runner-common.sh V="/var/run/github-runner" -A="\$$V-last-activity" -J="\$$V-jobs" -H="\$$V-has-run-job" - -[ ! -f "\$$A" ] && touch "\$$A" -L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) -N=\$$(date +%s) -I=\$$((N-L)) -[ -f "\$$H" ] && G=\$${RUNNER_GRACE_PERIOD:-60} || G=\$${RUNNER_INITIAL_GRACE_PERIOD:-180} -R=\$$(grep -l '"status":"running"' \$$J/*.job 2>/dev/null | wc -l || echo 0) - -if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then - log "TERMINATING: idle \$$I > grace \$$G" - # Deregister all runners - for RUNNER_DIR in $$homedir/runner-*; do - if [ -d "\$$RUNNER_DIR" ] && [ -f "\$$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in \$$RUNNER_DIR" - cd "\$$RUNNER_DIR" - pkill -INT -f "\$$RUNNER_DIR/run.sh" 2>/dev/null || true - sleep 1 - # Read token from stored file - if [ -f "\$$RUNNER_DIR/.runner-token" ]; then - TOKEN=\$$(cat "\$$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$$TOKEN 2>&1 - log "Deregistration exit: \$$?" - fi - fi - done +A="\\$$V-last-activity" +J="\\$$V-jobs" +H="\\$$V-has-run-job" +[ ! -f "\\$$A" ] && touch "\\$$A" +L=\\$$(stat -c %Y "\\$$A" 2>/dev/null || echo 0) +N=\\$$(date +%s) +I=\\$$((N-L)) +[ -f "\\$$H" ] && G=\\$${RUNNER_GRACE_PERIOD:-60} || G=\\$${RUNNER_INITIAL_GRACE_PERIOD:-180} +R=\\$$(grep -l '\"status\":\"running\"' \\$$J/*.job 2>/dev/null | wc -l || echo 0) +if [ \\$$R -eq 0 ] && [ \\$$I -gt \\$$G ]; then + log "TERMINATING: idle \\$$I > grace \\$$G" + deregister_all_runners flush_cloudwatch_logs - # In debug mode, give time for debugging before shutdown - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping 600 seconds before shutdown..." - sleep 600 - log "Debug period ended, shutting down" - else - log "Shutting down" - fi - sudo shutdown -h now + debug_sleep_and_shutdown else - [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" + [ \\$$R -gt 0 ] && log "\\$$R job(s) running" || log "Idle \\$$I/\\$$G sec" fi EOFT -chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh +chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh # Set up job tracking directory V="/var/run/github-runner" @@ -405,7 +269,7 @@ systemctl daemon-reload systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer -# Build metadata labels (using metadata fetched earlier) +# Build metadata labels (these will be added to the runner labels) METADATA_LABELS=",instance-id:$${INSTANCE_ID},instance-type:$${INSTANCE_TYPE}" if [ -n "$github_workflow" ]; then WORKFLOW_LABEL=$$(echo "$github_workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') @@ -414,104 +278,39 @@ fi [ -n "$github_run_id" ] && METADATA_LABELS="$${METADATA_LABELS},run-id:$github_run_id" [ -n "$github_run_number" ] && METADATA_LABELS="$${METADATA_LABELS},run-number:$github_run_number" -# Process each runner configuration log "Setting up $$RUNNERS_PER_INSTANCE runner(s)" -# Function to configure a single runner -# Export it so it's available to subprocesses -configure_runner() { - local idx=$$1 - local token=$$2 - local labels=$$3 - local homedir=$$4 - local repo=$$5 - local instance_id=$$6 - local runner_grace_period=$$7 - local runner_initial_grace_period=$$8 - - log "Configuring runner $$idx..." - - # Create runner directory - local runner_dir="$$homedir/runner-$$idx" - mkdir -p "$$runner_dir" - cd "$$runner_dir" - - # Extract runner - tar -xzf /tmp/runner.tar.gz - - # Install dependencies if needed (for AL2023, Debian 13, etc.) - if [ -f ./bin/installdependencies.sh ]; then - log "Installing runner dependencies..." - sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" - fi - - # Save token for deregistration - echo "$$token" > .runner-token - - # Create env file - cat > .env << EOF -ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh -ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh -RUNNER_HOME=$$runner_dir -RUNNER_INDEX=$$idx -RUNNER_GRACE_PERIOD=$$runner_grace_period -RUNNER_INITIAL_GRACE_PERIOD=$$runner_initial_grace_period -EOF - - # Configure runner - local runner_name="ec2-$$instance_id-$$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ - --url "https://github.com/$$repo" \ - --token "$$token" \ - --labels "$$labels" \ - --name "$$runner_name" \ - --disableupdate \ - --unattended 2>&1 | tee /tmp/runner-$$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$$idx-config.log; then - log "Runner $$idx registered successfully" - else - log_error "Failed to register runner $$idx" - return 1 - fi - - # Start runner in background - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & - local pid=$$! - log "Started runner $$idx in $$runner_dir (PID: $$pid)" - - return 0 -} -# Export the function so it's available to child processes +# Export functions for subprocesses (variables already exported from runner-common.sh) export -f configure_runner export -f log export -f log_error +export -f get_metadata +export -f flush_cloudwatch_logs +export -f deregister_all_runners +export -f debug_sleep_and_shutdown +export -f wait_for_dpkg_lock -# Parse simple delimited format: space-delimited tokens, pipe-delimited labels +# Parse space-delimited tokens and pipe-delimited labels IFS=' ' read -ra tokens <<< "$runner_tokens" IFS='|' read -ra labels <<< "$runner_labels" num_runners=$${#tokens[@]} log "Configuring $$num_runners runner(s) in parallel" -# Start configuration for each runner in background +# Start configuration for each runner in parallel pids=() for i in $${!tokens[@]}; do token=$${tokens[$$i]} - label=$${labels[$$i]:-} # Default to empty if no label - + label=$${labels[$$i]:-} if [ -z "$$token" ]; then log_error "No token for runner $$i" continue fi - - # Start configuration in background ( configure_runner $$i "$$token" "$${label}$$METADATA_LABELS" "$$homedir" "$repo" "$$INSTANCE_ID" "$runner_grace_period" "$runner_initial_grace_period" echo $$? > /tmp/runner-$$i-status ) & pids+=($$!) - log "Started configuration for runner $$i (PID: $${pids[-1]})" done @@ -537,15 +336,16 @@ fi log "All runners registered and started" touch /var/run/github-runner-registered -# Kill watchdog +# Kill registration watchdog now that runners are registered if [ -f /var/run/github-runner-watchdog.pid ]; then WATCHDOG_PID=$$(cat /var/run/github-runner-watchdog.pid) kill $$WATCHDOG_PID 2>/dev/null || true rm -f /var/run/github-runner-watchdog.pid fi +# Final setup - ensure runner directories are accessible for debugging touch /var/run/github-runner-started chmod o+x $$homedir for RUNNER_DIR in $$homedir/runner-*; do [ -d "$$RUNNER_DIR/_diag" ] && chmod 755 "$$RUNNER_DIR/_diag" -done +done \ No newline at end of file diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index edabb8a..b036e5c 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -46,35 +46,68 @@ exec 2> >(tee -a /var/log/runner-debug.log >&2) - if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then - echo "[DEBUG] Debug mode enabled - set -x active" >&2 - set -x + debug="" + + [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x + + homedir="/home/ec2-user" + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log + fi + fi + else + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log fi + export homedir + + cat > /usr/local/bin/runner-common.sh << EOSF + homedir="$homedir" + debug="$debug" + export homedir debug + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true - fi + wait_for_dpkg_lock() { + local timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done } - cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' - #!/bin/bash - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true + systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true fi } - EOCF get_metadata() { local path="$1" @@ -87,8 +120,36 @@ return 0 # Always return success to avoid set -e issues } - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + deregister_all_runners() { + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in $RUNNER_DIR" + cd "$RUNNER_DIR" + pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "$RUNNER_DIR/.runner-token" ]; then + TOKEN=$(cat "$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 + log "Deregistration exit: $?" + fi + fi + done + } + + debug_sleep_and_shutdown() { + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) + log "SSH into instance with: ssh ${ssh_user}@${public_ip}" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down immediately (debug mode not enabled)" + fi + shutdown -h now + } terminate_instance() { local reason="$1" @@ -108,17 +169,65 @@ fi flush_cloudwatch_logs + debug_sleep_and_shutdown + exit 1 + } + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 - log "Sleeping for 300 seconds before shutdown to allow debugging..." - log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 300 + log "Configuring runner $idx..." - log "Shutting down after debug delay" - shutdown -h now - exit 1 + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + tar -xzf /tmp/runner.tar.gz + + if [ -f ./bin/installdependencies.sh ]; then + log "Installing runner dependencies..." + sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + fi + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF + + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi + + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" + + return 0 } + EOSF + chmod +x /usr/local/bin/runner-common.sh + source /usr/local/bin/runner-common.sh + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR REGISTRATION_TIMEOUT="300" @@ -143,37 +252,6 @@ - homedir="/home/ec2-user" - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi - fi - else - log "Using: $homedir" - fi - exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" @@ -181,7 +259,6 @@ INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -191,22 +268,9 @@ if [ "" != "" ]; then log "Installing CloudWatch agent" ( - if command -v dpkg >/dev/null 2>&1; then log "Detected dpkg-based system" - - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done - + wait_for_dpkg_lock wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb @@ -218,55 +282,33 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - { - "agent": { - "run_as_user": "cwagent" - }, - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, - { "file_path": "/var/log/runner-debug.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, - { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, - { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, - { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } - ] - } - } - } - } + {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ + {"file_path":"/var/log/runner-setup.log","log_group_name":"","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, + {"file_path":"/var/log/runner-debug.log","log_group_name":"","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, + {"file_path":"/tmp/job-started-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, + {"file_path":"/tmp/job-completed-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, + {"file_path":"/tmp/termination-check.log","log_group_name":"","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, + {"file_path":"/tmp/runner-*-config.log","log_group_name":"","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + ]}}}} EOF - - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s - + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s log "CloudWatch agent started" ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" - + mkdir -p "$homedir/.ssh" + chmod 700 "$homedir/.ssh" echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" fi - log "SSH key added for user $DEFAULT_USER" fi @@ -298,6 +340,7 @@ terminate_instance "No download tool available" fi log "Downloaded runner binary" + cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 @@ -305,7 +348,7 @@ RUNNER_IDX="${RUNNER_INDEX:-0}" echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -322,43 +365,28 @@ cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh + source /usr/local/bin/runner-common.sh V="/var/run/github-runner" - A="\$V-last-activity" - J="\$V-jobs" - H="\$V-has-run-job" - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in \$RUNNER_DIR" - cd "\$RUNNER_DIR" - pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true - sleep 1 - if [ -f "\$RUNNER_DIR/.runner-token" ]; then - TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 - log "Deregistration exit: \$?" - fi - fi - done + A="\\$V-last-activity" + J="\\$V-jobs" + H="\\$V-has-run-job" + [ ! -f "\\$A" ] && touch "\\$A" + L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) + N=\\$(date +%s) + I=\\$((N-L)) + [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then + log "TERMINATING: idle \\$I > grace \\$G" + deregister_all_runners flush_cloudwatch_logs - log "Shutting down" - sudo shutdown -h now + debug_sleep_and_shutdown else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" + [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" fi EOFT - chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh V="/var/run/github-runner" mkdir -p $V-jobs @@ -400,60 +428,14 @@ log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - - tar -xzf /tmp/runner.tar.gz - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ - --url "https://github.com/$repo" \ - --token "$token" \ - --labels "$labels" \ - --name "$runner_name" \ - --disableupdate \ - --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } export -f configure_runner export -f log export -f log_error + export -f get_metadata + export -f flush_cloudwatch_logs + export -f deregister_all_runners + export -f debug_sleep_and_shutdown + export -f wait_for_dpkg_lock IFS=' ' read -ra tokens <<< "test" IFS='|' read -ra labels <<< "label" @@ -464,19 +446,16 @@ pids=() for i in ${!tokens[@]}; do token=${tokens[$i]} - label=${labels[$i]:-} # Default to empty if no label - + label=${labels[$i]:-} if [ -z "$token" ]; then log_error "No token for runner $i" continue fi - ( configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" done @@ -512,7 +491,6 @@ for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done - ''', }) # --- @@ -559,35 +537,68 @@ exec 2> >(tee -a /var/log/runner-debug.log >&2) - if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then - echo "[DEBUG] Debug mode enabled - set -x active" >&2 - set -x + debug="" + + [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x + + homedir="/home/ec2-user" + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log + fi + fi + else + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log fi + export homedir + + cat > /usr/local/bin/runner-common.sh << EOSF + homedir="$homedir" + debug="$debug" + export homedir debug + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true - fi + wait_for_dpkg_lock() { + local timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done } - cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' - #!/bin/bash - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true + systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true fi } - EOCF get_metadata() { local path="$1" @@ -600,8 +611,36 @@ return 0 # Always return success to avoid set -e issues } - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + deregister_all_runners() { + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in $RUNNER_DIR" + cd "$RUNNER_DIR" + pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "$RUNNER_DIR/.runner-token" ]; then + TOKEN=$(cat "$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 + log "Deregistration exit: $?" + fi + fi + done + } + + debug_sleep_and_shutdown() { + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) + log "SSH into instance with: ssh ${ssh_user}@${public_ip}" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down immediately (debug mode not enabled)" + fi + shutdown -h now + } terminate_instance() { local reason="$1" @@ -621,20 +660,68 @@ fi flush_cloudwatch_logs - - log "Sleeping for 300 seconds before shutdown to allow debugging..." - log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 300 - - log "Shutting down after debug delay" - shutdown -h now + debug_sleep_and_shutdown exit 1 } - trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - - REGISTRATION_TIMEOUT="300" + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 + + log "Configuring runner $idx..." + + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + tar -xzf /tmp/runner.tar.gz + + if [ -f ./bin/installdependencies.sh ]; then + log "Installing runner dependencies..." + sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + fi + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF + + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi + + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" + + return 0 + } + EOSF + chmod +x /usr/local/bin/runner-common.sh + source /usr/local/bin/runner-common.sh + + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + + REGISTRATION_TIMEOUT="300" if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" REGISTRATION_TIMEOUT=300 @@ -656,37 +743,6 @@ - homedir="/home/ec2-user" - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi - fi - else - log "Using: $homedir" - fi - exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" @@ -694,7 +750,6 @@ INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -704,22 +759,9 @@ if [ "" != "" ]; then log "Installing CloudWatch agent" ( - if command -v dpkg >/dev/null 2>&1; then log "Detected dpkg-based system" - - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done - + wait_for_dpkg_lock wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb @@ -731,55 +773,33 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - { - "agent": { - "run_as_user": "cwagent" - }, - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, - { "file_path": "/var/log/runner-debug.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, - { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, - { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, - { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } - ] - } - } - } - } + {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ + {"file_path":"/var/log/runner-setup.log","log_group_name":"","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, + {"file_path":"/var/log/runner-debug.log","log_group_name":"","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, + {"file_path":"/tmp/job-started-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, + {"file_path":"/tmp/job-completed-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, + {"file_path":"/tmp/termination-check.log","log_group_name":"","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, + {"file_path":"/tmp/runner-*-config.log","log_group_name":"","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + ]}}}} EOF - - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s - + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s log "CloudWatch agent started" ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" - + mkdir -p "$homedir/.ssh" + chmod 700 "$homedir/.ssh" echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" fi - log "SSH key added for user $DEFAULT_USER" fi @@ -811,6 +831,7 @@ terminate_instance "No download tool available" fi log "Downloaded runner binary" + cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 @@ -818,7 +839,7 @@ RUNNER_IDX="${RUNNER_INDEX:-0}" echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -835,43 +856,28 @@ cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh + source /usr/local/bin/runner-common.sh V="/var/run/github-runner" - A="\$V-last-activity" - J="\$V-jobs" - H="\$V-has-run-job" - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in \$RUNNER_DIR" - cd "\$RUNNER_DIR" - pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true - sleep 1 - if [ -f "\$RUNNER_DIR/.runner-token" ]; then - TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 - log "Deregistration exit: \$?" - fi - fi - done + A="\\$V-last-activity" + J="\\$V-jobs" + H="\\$V-has-run-job" + [ ! -f "\\$A" ] && touch "\\$A" + L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) + N=\\$(date +%s) + I=\\$((N-L)) + [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then + log "TERMINATING: idle \\$I > grace \\$G" + deregister_all_runners flush_cloudwatch_logs - log "Shutting down" - sudo shutdown -h now + debug_sleep_and_shutdown else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" + [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" fi EOFT - chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh V="/var/run/github-runner" mkdir -p $V-jobs @@ -913,60 +919,14 @@ log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - - tar -xzf /tmp/runner.tar.gz - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ - --url "https://github.com/$repo" \ - --token "$token" \ - --labels "$labels" \ - --name "$runner_name" \ - --disableupdate \ - --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } export -f configure_runner export -f log export -f log_error + export -f get_metadata + export -f flush_cloudwatch_logs + export -f deregister_all_runners + export -f debug_sleep_and_shutdown + export -f wait_for_dpkg_lock IFS=' ' read -ra tokens <<< "test" IFS='|' read -ra labels <<< "label" @@ -977,19 +937,16 @@ pids=() for i in ${!tokens[@]}; do token=${tokens[$i]} - label=${labels[$i]:-} # Default to empty if no label - + label=${labels[$i]:-} if [ -z "$token" ]; then log_error "No token for runner $i" continue fi - ( configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" done @@ -1025,7 +982,6 @@ for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done - ''', }) # --- @@ -1036,35 +992,68 @@ exec 2> >(tee -a /var/log/runner-debug.log >&2) - if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then - echo "[DEBUG] Debug mode enabled - set -x active" >&2 - set -x + debug="" + + [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x + + homedir="/home/ec2-user" + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log + fi + fi + else + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log fi + export homedir + + cat > /usr/local/bin/runner-common.sh << EOSF + homedir="$homedir" + debug="$debug" + export homedir debug + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true - fi + wait_for_dpkg_lock() { + local timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done } - cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' - #!/bin/bash - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true + systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true fi } - EOCF get_metadata() { local path="$1" @@ -1077,8 +1066,36 @@ return 0 # Always return success to avoid set -e issues } - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + deregister_all_runners() { + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in $RUNNER_DIR" + cd "$RUNNER_DIR" + pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "$RUNNER_DIR/.runner-token" ]; then + TOKEN=$(cat "$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 + log "Deregistration exit: $?" + fi + fi + done + } + + debug_sleep_and_shutdown() { + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) + log "SSH into instance with: ssh ${ssh_user}@${public_ip}" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down immediately (debug mode not enabled)" + fi + shutdown -h now + } terminate_instance() { local reason="$1" @@ -1098,17 +1115,65 @@ fi flush_cloudwatch_logs + debug_sleep_and_shutdown + exit 1 + } + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 - log "Sleeping for 300 seconds before shutdown to allow debugging..." - log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 300 + log "Configuring runner $idx..." - log "Shutting down after debug delay" - shutdown -h now - exit 1 + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + tar -xzf /tmp/runner.tar.gz + + if [ -f ./bin/installdependencies.sh ]; then + log "Installing runner dependencies..." + sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + fi + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF + + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi + + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" + + return 0 } + EOSF + chmod +x /usr/local/bin/runner-common.sh + source /usr/local/bin/runner-common.sh + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR REGISTRATION_TIMEOUT="300" @@ -1133,37 +1198,6 @@ - homedir="/home/ec2-user" - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi - fi - else - log "Using: $homedir" - fi - exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" @@ -1171,7 +1205,6 @@ INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -1181,22 +1214,9 @@ if [ "" != "" ]; then log "Installing CloudWatch agent" ( - if command -v dpkg >/dev/null 2>&1; then log "Detected dpkg-based system" - - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done - + wait_for_dpkg_lock wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb @@ -1208,55 +1228,33 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - { - "agent": { - "run_as_user": "cwagent" - }, - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { "file_path": "/var/log/runner-setup.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, - { "file_path": "/var/log/runner-debug.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, - { "file_path": "/tmp/job-started-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, - { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, - { "file_path": "/tmp/termination-check.log", "log_group_name": "", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } - ] - } - } - } - } + {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ + {"file_path":"/var/log/runner-setup.log","log_group_name":"","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, + {"file_path":"/var/log/runner-debug.log","log_group_name":"","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, + {"file_path":"/tmp/job-started-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, + {"file_path":"/tmp/job-completed-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, + {"file_path":"/tmp/termination-check.log","log_group_name":"","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, + {"file_path":"/tmp/runner-*-config.log","log_group_name":"","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + ]}}}} EOF - - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s - + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s log "CloudWatch agent started" ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" - + mkdir -p "$homedir/.ssh" + chmod 700 "$homedir/.ssh" echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" fi - log "SSH key added for user $DEFAULT_USER" fi @@ -1288,6 +1286,7 @@ terminate_instance "No download tool available" fi log "Downloaded runner binary" + cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 @@ -1295,7 +1294,7 @@ RUNNER_IDX="${RUNNER_INDEX:-0}" echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -1312,43 +1311,28 @@ cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh + source /usr/local/bin/runner-common.sh V="/var/run/github-runner" - A="\$V-last-activity" - J="\$V-jobs" - H="\$V-has-run-job" - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in \$RUNNER_DIR" - cd "\$RUNNER_DIR" - pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true - sleep 1 - if [ -f "\$RUNNER_DIR/.runner-token" ]; then - TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 - log "Deregistration exit: \$?" - fi - fi - done + A="\\$V-last-activity" + J="\\$V-jobs" + H="\\$V-has-run-job" + [ ! -f "\\$A" ] && touch "\\$A" + L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) + N=\\$(date +%s) + I=\\$((N-L)) + [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then + log "TERMINATING: idle \\$I > grace \\$G" + deregister_all_runners flush_cloudwatch_logs - log "Shutting down" - sudo shutdown -h now + debug_sleep_and_shutdown else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" + [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" fi EOFT - chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh V="/var/run/github-runner" mkdir -p $V-jobs @@ -1390,60 +1374,14 @@ log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - - tar -xzf /tmp/runner.tar.gz - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ - --url "https://github.com/$repo" \ - --token "$token" \ - --labels "$labels" \ - --name "$runner_name" \ - --disableupdate \ - --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } export -f configure_runner export -f log export -f log_error + export -f get_metadata + export -f flush_cloudwatch_logs + export -f deregister_all_runners + export -f debug_sleep_and_shutdown + export -f wait_for_dpkg_lock IFS=' ' read -ra tokens <<< "test" IFS='|' read -ra labels <<< "label" @@ -1454,19 +1392,16 @@ pids=() for i in ${!tokens[@]}; do token=${tokens[$i]} - label=${labels[$i]:-} # Default to empty if no label - + label=${labels[$i]:-} if [ -z "$token" ]; then log_error "No token for runner $i" continue fi - ( configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" done @@ -1502,7 +1437,6 @@ for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done - ''' # --- # name: test_build_user_data_with_cloudwatch @@ -1512,35 +1446,68 @@ exec 2> >(tee -a /var/log/runner-debug.log >&2) - if [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ]; then - echo "[DEBUG] Debug mode enabled - set -x active" >&2 - set -x + debug="" + + [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x + + homedir="/home/ec2-user" + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log + break + fi + done + + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log + fi + fi + else + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log fi + export homedir + + cat > /usr/local/bin/runner-common.sh << EOSF + homedir="$homedir" + debug="$debug" + export homedir debug + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true - fi + wait_for_dpkg_lock() { + local timeout=120 + while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + if [ $timeout -le 0 ]; then + log "WARNING: dpkg lock timeout, proceeding anyway" + break + fi + log "dpkg is locked, waiting... ($timeout seconds remaining)" + sleep 5 + timeout=$((timeout - 5)) + done } - cat > /usr/local/bin/runner-common-functions.sh << 'EOCF' - #!/bin/bash - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null \ - || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null \ - || true + systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true fi } - EOCF get_metadata() { local path="$1" @@ -1553,8 +1520,36 @@ return 0 # Always return success to avoid set -e issues } - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR + deregister_all_runners() { + for RUNNER_DIR in $homedir/runner-*; do + if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then + log "Deregistering runner in $RUNNER_DIR" + cd "$RUNNER_DIR" + pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + sleep 1 + if [ -f "$RUNNER_DIR/.runner-token" ]; then + TOKEN=$(cat "$RUNNER_DIR/.runner-token") + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 + log "Deregistration exit: $?" + fi + fi + done + } + + debug_sleep_and_shutdown() { + if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then + log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) + log "SSH into instance with: ssh ${ssh_user}@${public_ip}" + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + sleep 600 + log "Debug period ended, shutting down" + else + log "Shutting down immediately (debug mode not enabled)" + fi + shutdown -h now + } terminate_instance() { local reason="$1" @@ -1574,17 +1569,65 @@ fi flush_cloudwatch_logs + debug_sleep_and_shutdown + exit 1 + } + + configure_runner() { + local idx=$1 + local token=$2 + local labels=$3 + local homedir=$4 + local repo=$5 + local instance_id=$6 + local runner_grace_period=$7 + local runner_initial_grace_period=$8 - log "Sleeping for 300 seconds before shutdown to allow debugging..." - log "SSH into instance with: ssh ubuntu@$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 300 + log "Configuring runner $idx..." - log "Shutting down after debug delay" - shutdown -h now - exit 1 + local runner_dir="$homedir/runner-$idx" + mkdir -p "$runner_dir" + cd "$runner_dir" + tar -xzf /tmp/runner.tar.gz + + if [ -f ./bin/installdependencies.sh ]; then + log "Installing runner dependencies..." + sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + fi + + echo "$token" > .runner-token + + cat > .env << EOF + ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh + RUNNER_HOME=$runner_dir + RUNNER_INDEX=$idx + RUNNER_GRACE_PERIOD=$runner_grace_period + RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period + EOF + + local runner_name="ec2-$instance_id-$idx" + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log + + if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then + log "Runner $idx registered successfully" + else + log_error "Failed to register runner $idx" + return 1 + fi + + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + local pid=$! + log "Started runner $idx in $runner_dir (PID: $pid)" + + return 0 } + EOSF + chmod +x /usr/local/bin/runner-common.sh + source /usr/local/bin/runner-common.sh + logger "EC2-GHA: Starting userdata script" + trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR REGISTRATION_TIMEOUT="300" @@ -1609,37 +1652,6 @@ - homedir="/home/ec2-user" - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - log "Auto-detected: $homedir" - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - log "Using fallback: $homedir" - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - log "Detected: $homedir ($owner)" - fi - fi - else - log "Using: $homedir" - fi - exec >> /var/log/runner-setup.log 2>&1 log "Starting runner setup" @@ -1647,7 +1659,6 @@ INSTANCE_ID=$(get_metadata "instance-id") REGION=$(get_metadata "placement/region") AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" MAX_LIFETIME_MINUTES=360 @@ -1657,22 +1668,9 @@ if [ "/aws/ec2/github-runners" != "" ]; then log "Installing CloudWatch agent" ( - if command -v dpkg >/dev/null 2>&1; then log "Detected dpkg-based system" - - log "Waiting for dpkg lock to be released..." - timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" - sleep 5 - timeout=$((timeout - 5)) - done - + wait_for_dpkg_lock wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb @@ -1684,55 +1682,33 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - { - "agent": { - "run_as_user": "cwagent" - }, - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { "file_path": "/var/log/runner-setup.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-setup", "timezone": "UTC" }, - { "file_path": "/var/log/runner-debug.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-debug", "timezone": "UTC" }, - { "file_path": "/tmp/job-started-hook.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/job-started", "timezone": "UTC" }, - { "file_path": "/tmp/job-completed-hook.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, - { "file_path": "/tmp/termination-check.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/termination", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Runner_**.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/runner-diag", "timezone": "UTC" }, - { "file_path": "/home/ec2-user/_diag/Worker_**.log", "log_group_name": "/aws/ec2/github-runners", "log_stream_name": "{instance_id}/worker-diag", "timezone": "UTC" } - ] - } - } - } - } + {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ + {"file_path":"/var/log/runner-setup.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, + {"file_path":"/var/log/runner-debug.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, + {"file_path":"/tmp/job-started-hook.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, + {"file_path":"/tmp/job-completed-hook.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, + {"file_path":"/tmp/termination-check.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, + {"file_path":"/tmp/runner-*-config.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, + {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + ]}}}} EOF - - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ - -a fetch-config \ - -m ec2 \ - -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \ - -s - + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s log "CloudWatch agent started" ) || log "WARNING: CloudWatch agent installation failed, continuing without it" fi if [ -n "" ]; then log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - - mkdir -p "/home/ec2-user/.ssh" - chmod 700 "/home/ec2-user/.ssh" - + mkdir -p "$homedir/.ssh" + chmod 700 "$homedir/.ssh" echo "" >> "$homedir/.ssh/authorized_keys" chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" fi - log "SSH key added for user $DEFAULT_USER" fi @@ -1764,6 +1740,7 @@ terminate_instance "No download tool available" fi log "Downloaded runner binary" + cat > /usr/local/bin/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 @@ -1771,7 +1748,7 @@ RUNNER_IDX="${RUNNER_INDEX:-0}" echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" mkdir -p $V-jobs - echo '{"status":"running","runner":"'$RUNNER_IDX'"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job touch $V-last-activity $V-has-run-job EOFS @@ -1788,43 +1765,28 @@ cat > /usr/local/bin/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common-functions.sh + source /usr/local/bin/runner-common.sh V="/var/run/github-runner" - A="\$V-last-activity" - J="\$V-jobs" - H="\$V-has-run-job" - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "\$RUNNER_DIR" ] && [ -f "\$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in \$RUNNER_DIR" - cd "\$RUNNER_DIR" - pkill -INT -f "\$RUNNER_DIR/run.sh" 2>/dev/null || true - sleep 1 - if [ -f "\$RUNNER_DIR/.runner-token" ]; then - TOKEN=\$(cat "\$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token \$TOKEN 2>&1 - log "Deregistration exit: \$?" - fi - fi - done + A="\\$V-last-activity" + J="\\$V-jobs" + H="\\$V-has-run-job" + [ ! -f "\\$A" ] && touch "\\$A" + L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) + N=\\$(date +%s) + I=\\$((N-L)) + [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then + log "TERMINATING: idle \\$I > grace \\$G" + deregister_all_runners flush_cloudwatch_logs - log "Shutting down" - sudo shutdown -h now + debug_sleep_and_shutdown else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" + [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" fi EOFT - chmod +x /usr/local/bin/runner-common-functions.sh /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh V="/var/run/github-runner" mkdir -p $V-jobs @@ -1866,60 +1828,14 @@ log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - - tar -xzf /tmp/runner.tar.gz - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh \ - --url "https://github.com/$repo" \ - --token "$token" \ - --labels "$labels" \ - --name "$runner_name" \ - --disableupdate \ - --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } export -f configure_runner export -f log export -f log_error + export -f get_metadata + export -f flush_cloudwatch_logs + export -f deregister_all_runners + export -f debug_sleep_and_shutdown + export -f wait_for_dpkg_lock IFS=' ' read -ra tokens <<< "test" IFS='|' read -ra labels <<< "label" @@ -1930,19 +1846,16 @@ pids=() for i in ${!tokens[@]}; do token=${tokens[$i]} - label=${labels[$i]:-} # Default to empty if no label - + label=${labels[$i]:-} if [ -z "$token" ]; then log_error "No token for runner $i" continue fi - ( configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" done @@ -1978,6 +1891,5 @@ for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done - ''' # --- From 41c79f674082d433bc8d09c17297d2ad9d85b108 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 01:19:57 -0400 Subject: [PATCH 24/69] runner-common fix --- src/ec2_gha/templates/user-script.sh.templ | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index fbe07f5..0b44588 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -44,6 +44,7 @@ fi export homedir # Write shared functions that will be used by multiple scripts +# First write the variables that need template expansion cat > /usr/local/bin/runner-common.sh << EOSF # Auto-generated shared functions and variables # Set homedir for scripts that source this file @@ -51,8 +52,13 @@ homedir="$$homedir" debug="$$debug" export homedir debug +EOSF + +# Then append the shared functions using a quoted here-doc to prevent variable expansion +cat >> /usr/local/bin/runner-common.sh << 'EOSF' $shared_functions EOSF + chmod +x /usr/local/bin/runner-common.sh source /usr/local/bin/runner-common.sh From 7db6296eb01fe764be219cb0804535f5a8820b1d Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 01:42:12 -0400 Subject: [PATCH 25/69] gha-runner@rw/dns: print PublicDnsName to GHA UI --- pyproject.toml | 2 +- src/ec2_gha/start.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 847b047..b224296 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Start an AWS GitHub Actions Runner" readme = "README.md" requires-python = ">=3.12" authors = [{ name = "Ethan Holz", email = "ethan.holz@omsf.io" }] -dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@v1"] +dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@rw/dns"] [project.optional-dependencies] test = ["pytest", "pytest-cov", "moto[ec2]", "responses", "ruff", "syrupy"] diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index f525c8d..1a1279a 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -429,6 +429,34 @@ def wait_until_ready(self, ids: list[str], **kwargs): else: waiter.wait(InstanceIds=ids) + def get_instance_details(self, ids: list[str]) -> dict[str, dict]: + """Get instance details including DNS names. + + Parameters + ---------- + ids : list[str] + A list of instance IDs to get details for. + + Returns + ------- + dict[str, dict] + A dictionary mapping instance IDs to their details. + """ + ec2 = boto3.client("ec2", self.region_name) + response = ec2.describe_instances(InstanceIds=ids) + + details = {} + for reservation in response['Reservations']: + for instance in reservation['Instances']: + details[instance['InstanceId']] = { + 'PublicDnsName': instance.get('PublicDnsName', ''), + 'PublicIpAddress': instance.get('PublicIpAddress', ''), + 'PrivateIpAddress': instance.get('PrivateIpAddress', ''), + 'InstanceType': instance.get('InstanceType', ''), + 'State': instance['State']['Name'] + } + return details + def set_instance_mapping(self, mapping: dict[str, str]): """Set the instance mapping. From f76b72cb65cc6a325f05688abcbbe9c3da713729 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 01:42:18 -0400 Subject: [PATCH 26/69] azl deps fix --- .../templates/shared-functions.sh.templ | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/ec2_gha/templates/shared-functions.sh.templ b/src/ec2_gha/templates/shared-functions.sh.templ index 101627f..dbf00d7 100644 --- a/src/ec2_gha/templates/shared-functions.sh.templ +++ b/src/ec2_gha/templates/shared-functions.sh.templ @@ -121,7 +121,24 @@ configure_runner() { # Install dependencies if needed (for AL2023, Debian 13, etc.) if [ -f ./bin/installdependencies.sh ]; then log "Installing runner dependencies..." - sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + # Try to install dependencies, but don't fail if the script has issues + # Some distros may have the dependencies pre-installed or named differently + if ! sudo ./bin/installdependencies.sh >/dev/null 2>&1; then + log "Warning: installdependencies.sh reported issues, checking for libicu manually..." + # Try to install libicu manually based on the distro + if command -v yum >/dev/null 2>&1; then + # Amazon Linux, RHEL, CentOS + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + # Debian, Ubuntu + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + elif command -v dnf >/dev/null 2>&1; then + # Fedora, newer RHEL + sudo dnf install -y libicu >/dev/null 2>&1 || true + fi + fi fi # Save token for deregistration From 5b81d71ab9a2dc89fa365cdd2d858d476d5531be Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 01:53:15 -0400 Subject: [PATCH 27/69] compress templ --- .../templates/shared-functions.sh.templ | 17 ++--- src/ec2_gha/templates/user-script.sh.templ | 62 +++++++++++-------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/src/ec2_gha/templates/shared-functions.sh.templ b/src/ec2_gha/templates/shared-functions.sh.templ index dbf00d7..fa5063f 100644 --- a/src/ec2_gha/templates/shared-functions.sh.templ +++ b/src/ec2_gha/templates/shared-functions.sh.templ @@ -60,7 +60,7 @@ deregister_all_runners() { # Function to handle debug mode sleep and shutdown debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + log "Debug: Sleeping 600s before shutdown..." # Detect the SSH user from the home directory local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) @@ -118,25 +118,16 @@ configure_runner() { cd "$runner_dir" tar -xzf /tmp/runner.tar.gz - # Install dependencies if needed (for AL2023, Debian 13, etc.) + # Install dependencies if [ -f ./bin/installdependencies.sh ]; then - log "Installing runner dependencies..." - # Try to install dependencies, but don't fail if the script has issues - # Some distros may have the dependencies pre-installed or named differently + log "Installing dependencies..." if ! sudo ./bin/installdependencies.sh >/dev/null 2>&1; then - log "Warning: installdependencies.sh reported issues, checking for libicu manually..." - # Try to install libicu manually based on the distro + # Try libicu on failure if command -v yum >/dev/null 2>&1; then - # Amazon Linux, RHEL, CentOS sudo yum install -y libicu >/dev/null 2>&1 || true elif command -v apt-get >/dev/null 2>&1; then - # Debian, Ubuntu wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true - elif command -v dnf >/dev/null 2>&1; then - # Fedora, newer RHEL - sudo dnf install -y libicu >/dev/null 2>&1 || true fi fi fi diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 0b44588..f65b019 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -43,9 +43,13 @@ else fi export homedir +# Set common paths +B=/usr/local/bin +V=/var/run/github-runner + # Write shared functions that will be used by multiple scripts # First write the variables that need template expansion -cat > /usr/local/bin/runner-common.sh << EOSF +cat > $$B/runner-common.sh << EOSF # Auto-generated shared functions and variables # Set homedir for scripts that source this file homedir="$$homedir" @@ -55,12 +59,12 @@ export homedir debug EOSF # Then append the shared functions using a quoted here-doc to prevent variable expansion -cat >> /usr/local/bin/runner-common.sh << 'EOSF' +cat >> $$B/runner-common.sh << 'EOSF' $shared_functions EOSF -chmod +x /usr/local/bin/runner-common.sh -source /usr/local/bin/runner-common.sh +chmod +x $$B/runner-common.sh +source $$B/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR @@ -77,7 +81,7 @@ logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" ( log "Watchdog: Starting $$REGISTRATION_TIMEOUT second timeout" sleep $$REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then + if [ ! -f $$V-registered ]; then log "Watchdog: Registration marker not found after timeout" terminate_instance "Runner failed to register within $$REGISTRATION_TIMEOUT seconds" else @@ -86,7 +90,7 @@ logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" ) & REGISTRATION_WATCHDOG_PID=$$! log "Watchdog PID: $$REGISTRATION_WATCHDOG_PID" -echo $$REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid +echo $$REGISTRATION_WATCHDOG_PID > $$V-watchdog.pid # Run any custom user data script provided by the user $userdata @@ -128,16 +132,21 @@ if [ "$cloudwatch_logs_group" != "" ]; then else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi + # Build CloudWatch config with factored strings + P='"file_path":' + G=',"log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$$homedir" cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ -{"file_path":"/var/log/runner-setup.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, -{"file_path":"/var/log/runner-debug.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, -{"file_path":"/tmp/job-started-hook.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, -{"file_path":"/tmp/job-completed-hook.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, -{"file_path":"/tmp/termination-check.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, -{"file_path":"/tmp/runner-*-config.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, -{"file_path":"$homedir/_diag/Runner_**.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, -{"file_path":"$homedir/_diag/Worker_**.log","log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} +{$$P"/var/log/runner-setup.log"$${G}runner-setup$$Z, +{$$P"/var/log/runner-debug.log"$${G}runner-debug$$Z, +{$$P"/tmp/job-started-hook.log"$${G}job-started$$Z, +{$$P"/tmp/job-completed-hook.log"$${G}job-completed$$Z, +{$$P"/tmp/termination-check.log"$${G}termination$$Z, +{$$P"/tmp/runner-*-config.log"$${G}runner-config$$Z, +{$$P"$$H/_diag/Runner_**.log"$${G}runner-diag$$Z, +{$$P"$$H/_diag/Worker_**.log"$${G}worker-diag$$Z ]}}}} EOF /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s @@ -194,7 +203,7 @@ log "Downloaded runner binary" # Create job tracking scripts - these are called by GitHub runner hooks # job-started-hook.sh is called when a job starts -cat > /usr/local/bin/job-started-hook.sh << 'EOFS' +cat > $$B/job-started-hook.sh << 'EOFS' #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 V="/var/run/github-runner" @@ -206,7 +215,7 @@ touch $$V-last-activity $$V-has-run-job EOFS # job-completed-hook.sh is called when a job completes -cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' +cat > $$B/job-completed-hook.sh << 'EOFC' #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 V="/var/run/github-runner" @@ -217,10 +226,10 @@ touch $$V-last-activity EOFC # check-runner-termination.sh is called periodically to check if the instance should terminate -cat > /usr/local/bin/check-runner-termination.sh << EOFT +cat > $$B/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 -source /usr/local/bin/runner-common.sh +source $$B/runner-common.sh V="/var/run/github-runner" A="\\$$V-last-activity" J="\\$$V-jobs" @@ -241,10 +250,9 @@ else fi EOFT -chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh +chmod +x $$B/job-started-hook.sh $$B/job-completed-hook.sh $$B/check-runner-termination.sh # Set up job tracking directory -V="/var/run/github-runner" mkdir -p $$V-jobs touch $$V-last-activity @@ -257,7 +265,7 @@ After=network.target Type=oneshot Environment="RUNNER_GRACE_PERIOD=$runner_grace_period" Environment="RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" -ExecStart=/usr/local/bin/check-runner-termination.sh +ExecStart=$$B/check-runner-termination.sh EOF cat > /etc/systemd/system/runner-termination-check.timer << EOF @@ -340,18 +348,18 @@ if [ $$failed -ne 0 ]; then fi log "All runners registered and started" -touch /var/run/github-runner-registered +touch $$V-registered # Kill registration watchdog now that runners are registered -if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$$(cat /var/run/github-runner-watchdog.pid) +if [ -f $$V-watchdog.pid ]; then + WATCHDOG_PID=$$(cat $$V-watchdog.pid) kill $$WATCHDOG_PID 2>/dev/null || true - rm -f /var/run/github-runner-watchdog.pid + rm -f $$V-watchdog.pid fi # Final setup - ensure runner directories are accessible for debugging -touch /var/run/github-runner-started +touch $$V-started chmod o+x $$homedir for RUNNER_DIR in $$homedir/runner-*; do [ -d "$$RUNNER_DIR/_diag" ] && chmod 755 "$$RUNNER_DIR/_diag" -done \ No newline at end of file +done From 4dc62f078a73a3f4e42158c330552e383230b956 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 02:30:39 -0400 Subject: [PATCH 28/69] err handling --- .../templates/shared-functions.sh.templ | 20 ++++++++++++++++--- src/ec2_gha/templates/user-script.sh.templ | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/ec2_gha/templates/shared-functions.sh.templ b/src/ec2_gha/templates/shared-functions.sh.templ index fa5063f..589b8e1 100644 --- a/src/ec2_gha/templates/shared-functions.sh.templ +++ b/src/ec2_gha/templates/shared-functions.sh.templ @@ -121,12 +121,26 @@ configure_runner() { # Install dependencies if [ -f ./bin/installdependencies.sh ]; then log "Installing dependencies..." - if ! sudo ./bin/installdependencies.sh >/dev/null 2>&1; then - # Try libicu on failure - if command -v yum >/dev/null 2>&1; then + # Don't let installdependencies.sh failure trigger ERR trap + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing required packages manually..." + # Install runner dependencies based on package manager + if command -v dnf >/dev/null 2>&1; then + # Fedora/RHEL 8+/AL2023 + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + # RHEL 7/CentOS 7/AL2 sudo yum install -y libicu >/dev/null 2>&1 || true + # Note: lttng-ust often not available in standard repos for AL2 elif command -v apt-get >/dev/null 2>&1; then + # Debian/Ubuntu wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true fi fi diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index f65b019..687c204 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -321,6 +321,8 @@ for i in $${!tokens[@]}; do continue fi ( + # Override ERR trap in subshell to prevent global side effects + trap 'echo "Subshell error on line $$LINENO" >&2; exit 1' ERR configure_runner $$i "$$token" "$${label}$$METADATA_LABELS" "$$homedir" "$repo" "$$INSTANCE_ID" "$runner_grace_period" "$runner_initial_grace_period" echo $$? > /tmp/runner-$$i-status ) & From b9e574316493b2c713b60f38ae13de87e02a1aba Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 09:39:35 -0400 Subject: [PATCH 29/69] allow some runners to proceed, even if others fail to init --- src/ec2_gha/templates/user-script.sh.templ | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 687c204..326e31f 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -69,6 +69,8 @@ source $$B/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR trap 'terminate_instance "Setup script failed with error on line $$LINENO"' ERR +# Handle watchdog termination signal +trap 'if [ -f $$V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM # Set up registration timeout failsafe - terminate if runner doesn't register in time REGISTRATION_TIMEOUT="$runner_registration_timeout" @@ -78,15 +80,21 @@ if ! [[ "$$REGISTRATION_TIMEOUT" =~ ^[0-9]+$$ ]]; then REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" +# Create a marker file for watchdog termination request +touch $$V-watchdog-active ( log "Watchdog: Starting $$REGISTRATION_TIMEOUT second timeout" sleep $$REGISTRATION_TIMEOUT if [ ! -f $$V-registered ]; then log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $$REGISTRATION_TIMEOUT seconds" + # Signal main process to terminate instead of doing it directly + touch $$V-watchdog-terminate + # Kill the main script process to trigger its ERR trap + kill -TERM $$$$ 2>/dev/null || true else log "Watchdog: Registration marker found, exiting normally" fi + rm -f $$V-watchdog-active ) & REGISTRATION_WATCHDOG_PID=$$! log "Watchdog PID: $$REGISTRATION_WATCHDOG_PID" @@ -333,6 +341,7 @@ done # Wait for all background jobs to complete log "Waiting for all runner configurations to complete..." failed=0 +succeeded=0 for i in $${!pids[@]}; do wait $${pids[$$i]} if [ -f /tmp/runner-$$i-status ]; then @@ -340,17 +349,26 @@ for i in $${!pids[@]}; do rm -f /tmp/runner-$$i-status if [ "$$status" != "0" ]; then log_error "Runner $$i configuration failed" - failed=1 + failed=$$((failed + 1)) + else + succeeded=$$((succeeded + 1)) fi fi done -if [ $$failed -ne 0 ]; then - terminate_instance "One or more runners failed to register" +# Allow partial success - only terminate if ALL runners failed +if [ $$succeeded -eq 0 ] && [ $$failed -gt 0 ]; then + terminate_instance "All runners failed to register" +elif [ $$failed -gt 0 ]; then + log "WARNING: $$failed runner(s) failed, but $$succeeded succeeded. Continuing with partial capacity." fi -log "All runners registered and started" -touch $$V-registered +if [ $$succeeded -gt 0 ]; then + log "$$succeeded runner(s) registered and started successfully" + touch $$V-registered +else + log_error "No runners registered successfully" +fi # Kill registration watchdog now that runners are registered if [ -f $$V-watchdog.pid ]; then From 147526b4104821c213f2009c0be7a49a7de0f2e6 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 11:10:32 -0400 Subject: [PATCH 30/69] heredoc fixes --- src/ec2_gha/templates/user-script.sh.templ | 45 +- tests/__snapshots__/test_start.ambr | 592 +++++++++++++-------- 2 files changed, 393 insertions(+), 244 deletions(-) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 326e31f..85ed2f0 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -89,7 +89,7 @@ touch $$V-watchdog-active log "Watchdog: Registration marker not found after timeout" # Signal main process to terminate instead of doing it directly touch $$V-watchdog-terminate - # Kill the main script process to trigger its ERR trap + # Kill the main script process to trigger its TERM trap kill -TERM $$$$ 2>/dev/null || true else log "Watchdog: Registration marker found, exiting normally" @@ -211,25 +211,23 @@ log "Downloaded runner binary" # Create job tracking scripts - these are called by GitHub runner hooks # job-started-hook.sh is called when a job starts -cat > $$B/job-started-hook.sh << 'EOFS' +cat > $$B/job-started-hook.sh << EOFS #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 -V="/var/run/github-runner" -RUNNER_IDX="$${RUNNER_INDEX:-0}" -echo "[$$(date)] Runner-$$RUNNER_IDX: ${log_prefix_job_started} $${GITHUB_JOB}" +I="\$${RUNNER_INDEX:-0}" +echo "[\$$(date)] Runner-\$$I: ${log_prefix_job_started} \$${GITHUB_JOB}" mkdir -p $$V-jobs -echo '{\"status\":\"running\",\"runner\":\"'$$RUNNER_IDX'\"}' > $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job +echo '{"status":"running","runner":"'\$$I'"}' > $$V-jobs/\$${GITHUB_RUN_ID}-\$${GITHUB_JOB}-\$$I.job touch $$V-last-activity $$V-has-run-job EOFS # job-completed-hook.sh is called when a job completes -cat > $$B/job-completed-hook.sh << 'EOFC' +cat > $$B/job-completed-hook.sh << EOFC #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 -V="/var/run/github-runner" -RUNNER_IDX="$${RUNNER_INDEX:-0}" -echo "[$$(date)] Runner-$$RUNNER_IDX: ${log_prefix_job_completed} $${GITHUB_JOB}" -rm -f $$V-jobs/$${GITHUB_RUN_ID}-$${GITHUB_JOB}-$$RUNNER_IDX.job +I="\$${RUNNER_INDEX:-0}" +echo "[\$$(date)] Runner-\$$I: ${log_prefix_job_completed} \$${GITHUB_JOB}" +rm -f $$V-jobs/\$${GITHUB_RUN_ID}-\$${GITHUB_JOB}-\$$I.job touch $$V-last-activity EOFC @@ -238,23 +236,22 @@ cat > $$B/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 source $$B/runner-common.sh -V="/var/run/github-runner" -A="\\$$V-last-activity" -J="\\$$V-jobs" -H="\\$$V-has-run-job" -[ ! -f "\\$$A" ] && touch "\\$$A" -L=\\$$(stat -c %Y "\\$$A" 2>/dev/null || echo 0) -N=\\$$(date +%s) -I=\\$$((N-L)) -[ -f "\\$$H" ] && G=\\$${RUNNER_GRACE_PERIOD:-60} || G=\\$${RUNNER_INITIAL_GRACE_PERIOD:-180} -R=\\$$(grep -l '\"status\":\"running\"' \\$$J/*.job 2>/dev/null | wc -l || echo 0) -if [ \\$$R -eq 0 ] && [ \\$$I -gt \\$$G ]; then - log "TERMINATING: idle \\$$I > grace \\$$G" +A="$$V-last-activity" +J="$$V-jobs" +H="$$V-has-run-job" +[ ! -f "\$$A" ] && touch "\$$A" +L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) +N=\$$(date +%s) +I=\$$((N-L)) +[ -f "\$$H" ] && G=\$${RUNNER_GRACE_PERIOD:-60} || G=\$${RUNNER_INITIAL_GRACE_PERIOD:-180} +R=\$$(grep -l '"status":"running"' \$$J/*.job 2>/dev/null | wc -l || echo 0) +if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then + log "TERMINATING: idle \$$I > grace \$$G" deregister_all_runners flush_cloudwatch_logs debug_sleep_and_shutdown else - [ \\$$R -gt 0 ] && log "\\$$R job(s) running" || log "Idle \\$$I/\\$$G sec" + [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" fi EOFT diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index b036e5c..71da414 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -59,7 +59,7 @@ break fi done - + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do if [ -d "$dir" ]; then @@ -80,11 +80,17 @@ fi export homedir - cat > /usr/local/bin/runner-common.sh << EOSF + B=/usr/local/bin + V=/var/run/github-runner + + cat > $B/runner-common.sh << EOSF homedir="$homedir" debug="$debug" export homedir debug + EOSF + + cat >> $B/runner-common.sh << 'EOSF' log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -138,7 +144,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + log "Debug: Sleeping 600s before shutdown..." local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" @@ -191,8 +197,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing runner dependencies..." - sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing required packages manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi + fi fi echo "$token" > .runner-token @@ -223,12 +245,14 @@ return 0 } EOSF - chmod +x /usr/local/bin/runner-common.sh - source /usr/local/bin/runner-common.sh + + chmod +x $B/runner-common.sh + source $B/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM REGISTRATION_TIMEOUT="300" if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then @@ -236,19 +260,22 @@ REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-active ( log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then + if [ ! -f $V-registered ]; then log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-terminate + kill -TERM $$ 2>/dev/null || true else log "Watchdog: Registration marker found, exiting normally" fi + rm -f $V-watchdog-active ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid @@ -282,16 +309,20 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi + P='"file_path":' + G=',"log_group_name":"","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$homedir" cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {"file_path":"/var/log/runner-setup.log","log_group_name":"","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, - {"file_path":"/var/log/runner-debug.log","log_group_name":"","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, - {"file_path":"/tmp/job-started-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, - {"file_path":"/tmp/job-completed-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, - {"file_path":"/tmp/termination-check.log","log_group_name":"","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, - {"file_path":"/tmp/runner-*-config.log","log_group_name":"","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + {$P"/var/log/runner-setup.log"${G}runner-setup$Z, + {$P"/var/log/runner-debug.log"${G}runner-debug$Z, + {$P"/tmp/job-started-hook.log"${G}job-started$Z, + {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, + {$P"/tmp/termination-check.log"${G}termination$Z, + {$P"/tmp/runner-*-config.log"${G}runner-config$Z, + {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, + {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z ]}}}} EOF /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s @@ -341,54 +372,50 @@ fi log "Downloaded runner binary" - cat > /usr/local/bin/job-started-hook.sh << 'EOFS' + cat > $B/job-started-hook.sh << EOFS #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" mkdir -p $V-jobs - echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity $V-has-run-job EOFS - cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' + cat > $B/job-completed-hook.sh << EOFC #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" + rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << EOFT + cat > $B/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common.sh - V="/var/run/github-runner" - A="\\$V-last-activity" - J="\\$V-jobs" - H="\\$V-has-run-job" - [ ! -f "\\$A" ] && touch "\\$A" - L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) - N=\\$(date +%s) - I=\\$((N-L)) - [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then - log "TERMINATING: idle \\$I > grace \\$G" + source $B/runner-common.sh + A="$V-last-activity" + J="$V-jobs" + H="$V-has-run-job" + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" deregister_all_runners flush_cloudwatch_logs debug_sleep_and_shutdown else - [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT - chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - V="/var/run/github-runner" mkdir -p $V-jobs touch $V-last-activity @@ -400,7 +427,7 @@ Type=oneshot Environment="RUNNER_GRACE_PERIOD=61" Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=/usr/local/bin/check-runner-termination.sh + ExecStart=$B/check-runner-termination.sh EOF cat > /etc/systemd/system/runner-termination-check.timer << EOF @@ -452,6 +479,7 @@ continue fi ( + trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & @@ -461,6 +489,7 @@ log "Waiting for all runner configurations to complete..." failed=0 + succeeded=0 for i in ${!pids[@]}; do wait ${pids[$i]} if [ -f /tmp/runner-$i-status ]; then @@ -468,29 +497,38 @@ rm -f /tmp/runner-$i-status if [ "$status" != "0" ]; then log_error "Runner $i configuration failed" - failed=1 + failed=$((failed + 1)) + else + succeeded=$((succeeded + 1)) fi fi done - if [ $failed -ne 0 ]; then - terminate_instance "One or more runners failed to register" + if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then + terminate_instance "All runners failed to register" + elif [ $failed -gt 0 ]; then + log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." fi - log "All runners registered and started" - touch /var/run/github-runner-registered + if [ $succeeded -gt 0 ]; then + log "$succeeded runner(s) registered and started successfully" + touch $V-registered + else + log_error "No runners registered successfully" + fi - if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if [ -f $V-watchdog.pid ]; then + WATCHDOG_PID=$(cat $V-watchdog.pid) kill $WATCHDOG_PID 2>/dev/null || true - rm -f /var/run/github-runner-watchdog.pid + rm -f $V-watchdog.pid fi - touch /var/run/github-runner-started + touch $V-started chmod o+x $homedir for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done + ''', }) # --- @@ -550,7 +588,7 @@ break fi done - + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do if [ -d "$dir" ]; then @@ -571,11 +609,17 @@ fi export homedir - cat > /usr/local/bin/runner-common.sh << EOSF + B=/usr/local/bin + V=/var/run/github-runner + + cat > $B/runner-common.sh << EOSF homedir="$homedir" debug="$debug" export homedir debug + EOSF + + cat >> $B/runner-common.sh << 'EOSF' log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -629,7 +673,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + log "Debug: Sleeping 600s before shutdown..." local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" @@ -682,8 +726,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing runner dependencies..." - sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing required packages manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi + fi fi echo "$token" > .runner-token @@ -714,12 +774,14 @@ return 0 } EOSF - chmod +x /usr/local/bin/runner-common.sh - source /usr/local/bin/runner-common.sh + + chmod +x $B/runner-common.sh + source $B/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM REGISTRATION_TIMEOUT="300" if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then @@ -727,19 +789,22 @@ REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-active ( log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then + if [ ! -f $V-registered ]; then log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-terminate + kill -TERM $$ 2>/dev/null || true else log "Watchdog: Registration marker found, exiting normally" fi + rm -f $V-watchdog-active ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid @@ -773,16 +838,20 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi + P='"file_path":' + G=',"log_group_name":"","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$homedir" cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {"file_path":"/var/log/runner-setup.log","log_group_name":"","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, - {"file_path":"/var/log/runner-debug.log","log_group_name":"","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, - {"file_path":"/tmp/job-started-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, - {"file_path":"/tmp/job-completed-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, - {"file_path":"/tmp/termination-check.log","log_group_name":"","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, - {"file_path":"/tmp/runner-*-config.log","log_group_name":"","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + {$P"/var/log/runner-setup.log"${G}runner-setup$Z, + {$P"/var/log/runner-debug.log"${G}runner-debug$Z, + {$P"/tmp/job-started-hook.log"${G}job-started$Z, + {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, + {$P"/tmp/termination-check.log"${G}termination$Z, + {$P"/tmp/runner-*-config.log"${G}runner-config$Z, + {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, + {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z ]}}}} EOF /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s @@ -832,54 +901,50 @@ fi log "Downloaded runner binary" - cat > /usr/local/bin/job-started-hook.sh << 'EOFS' + cat > $B/job-started-hook.sh << EOFS #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" mkdir -p $V-jobs - echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity $V-has-run-job EOFS - cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' + cat > $B/job-completed-hook.sh << EOFC #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" + rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << EOFT + cat > $B/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common.sh - V="/var/run/github-runner" - A="\\$V-last-activity" - J="\\$V-jobs" - H="\\$V-has-run-job" - [ ! -f "\\$A" ] && touch "\\$A" - L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) - N=\\$(date +%s) - I=\\$((N-L)) - [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then - log "TERMINATING: idle \\$I > grace \\$G" + source $B/runner-common.sh + A="$V-last-activity" + J="$V-jobs" + H="$V-has-run-job" + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" deregister_all_runners flush_cloudwatch_logs debug_sleep_and_shutdown else - [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT - chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - V="/var/run/github-runner" mkdir -p $V-jobs touch $V-last-activity @@ -891,7 +956,7 @@ Type=oneshot Environment="RUNNER_GRACE_PERIOD=61" Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=/usr/local/bin/check-runner-termination.sh + ExecStart=$B/check-runner-termination.sh EOF cat > /etc/systemd/system/runner-termination-check.timer << EOF @@ -943,6 +1008,7 @@ continue fi ( + trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & @@ -952,6 +1018,7 @@ log "Waiting for all runner configurations to complete..." failed=0 + succeeded=0 for i in ${!pids[@]}; do wait ${pids[$i]} if [ -f /tmp/runner-$i-status ]; then @@ -959,29 +1026,38 @@ rm -f /tmp/runner-$i-status if [ "$status" != "0" ]; then log_error "Runner $i configuration failed" - failed=1 + failed=$((failed + 1)) + else + succeeded=$((succeeded + 1)) fi fi done - if [ $failed -ne 0 ]; then - terminate_instance "One or more runners failed to register" + if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then + terminate_instance "All runners failed to register" + elif [ $failed -gt 0 ]; then + log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." fi - log "All runners registered and started" - touch /var/run/github-runner-registered + if [ $succeeded -gt 0 ]; then + log "$succeeded runner(s) registered and started successfully" + touch $V-registered + else + log_error "No runners registered successfully" + fi - if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if [ -f $V-watchdog.pid ]; then + WATCHDOG_PID=$(cat $V-watchdog.pid) kill $WATCHDOG_PID 2>/dev/null || true - rm -f /var/run/github-runner-watchdog.pid + rm -f $V-watchdog.pid fi - touch /var/run/github-runner-started + touch $V-started chmod o+x $homedir for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done + ''', }) # --- @@ -1005,7 +1081,7 @@ break fi done - + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do if [ -d "$dir" ]; then @@ -1026,11 +1102,17 @@ fi export homedir - cat > /usr/local/bin/runner-common.sh << EOSF + B=/usr/local/bin + V=/var/run/github-runner + + cat > $B/runner-common.sh << EOSF homedir="$homedir" debug="$debug" export homedir debug + EOSF + + cat >> $B/runner-common.sh << 'EOSF' log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -1084,7 +1166,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + log "Debug: Sleeping 600s before shutdown..." local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" @@ -1137,8 +1219,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing runner dependencies..." - sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing required packages manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi + fi fi echo "$token" > .runner-token @@ -1169,12 +1267,14 @@ return 0 } EOSF - chmod +x /usr/local/bin/runner-common.sh - source /usr/local/bin/runner-common.sh + + chmod +x $B/runner-common.sh + source $B/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM REGISTRATION_TIMEOUT="300" if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then @@ -1182,19 +1282,22 @@ REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-active ( log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then + if [ ! -f $V-registered ]; then log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-terminate + kill -TERM $$ 2>/dev/null || true else log "Watchdog: Registration marker found, exiting normally" fi + rm -f $V-watchdog-active ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid @@ -1228,16 +1331,20 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi + P='"file_path":' + G=',"log_group_name":"","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$homedir" cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {"file_path":"/var/log/runner-setup.log","log_group_name":"","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, - {"file_path":"/var/log/runner-debug.log","log_group_name":"","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, - {"file_path":"/tmp/job-started-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, - {"file_path":"/tmp/job-completed-hook.log","log_group_name":"","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, - {"file_path":"/tmp/termination-check.log","log_group_name":"","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, - {"file_path":"/tmp/runner-*-config.log","log_group_name":"","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + {$P"/var/log/runner-setup.log"${G}runner-setup$Z, + {$P"/var/log/runner-debug.log"${G}runner-debug$Z, + {$P"/tmp/job-started-hook.log"${G}job-started$Z, + {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, + {$P"/tmp/termination-check.log"${G}termination$Z, + {$P"/tmp/runner-*-config.log"${G}runner-config$Z, + {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, + {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z ]}}}} EOF /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s @@ -1287,54 +1394,50 @@ fi log "Downloaded runner binary" - cat > /usr/local/bin/job-started-hook.sh << 'EOFS' + cat > $B/job-started-hook.sh << EOFS #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" mkdir -p $V-jobs - echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity $V-has-run-job EOFS - cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' + cat > $B/job-completed-hook.sh << EOFC #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" + rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << EOFT + cat > $B/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common.sh - V="/var/run/github-runner" - A="\\$V-last-activity" - J="\\$V-jobs" - H="\\$V-has-run-job" - [ ! -f "\\$A" ] && touch "\\$A" - L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) - N=\\$(date +%s) - I=\\$((N-L)) - [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then - log "TERMINATING: idle \\$I > grace \\$G" + source $B/runner-common.sh + A="$V-last-activity" + J="$V-jobs" + H="$V-has-run-job" + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" deregister_all_runners flush_cloudwatch_logs debug_sleep_and_shutdown else - [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT - chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - V="/var/run/github-runner" mkdir -p $V-jobs touch $V-last-activity @@ -1346,7 +1449,7 @@ Type=oneshot Environment="RUNNER_GRACE_PERIOD=61" Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=/usr/local/bin/check-runner-termination.sh + ExecStart=$B/check-runner-termination.sh EOF cat > /etc/systemd/system/runner-termination-check.timer << EOF @@ -1398,6 +1501,7 @@ continue fi ( + trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & @@ -1407,6 +1511,7 @@ log "Waiting for all runner configurations to complete..." failed=0 + succeeded=0 for i in ${!pids[@]}; do wait ${pids[$i]} if [ -f /tmp/runner-$i-status ]; then @@ -1414,29 +1519,38 @@ rm -f /tmp/runner-$i-status if [ "$status" != "0" ]; then log_error "Runner $i configuration failed" - failed=1 + failed=$((failed + 1)) + else + succeeded=$((succeeded + 1)) fi fi done - if [ $failed -ne 0 ]; then - terminate_instance "One or more runners failed to register" + if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then + terminate_instance "All runners failed to register" + elif [ $failed -gt 0 ]; then + log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." fi - log "All runners registered and started" - touch /var/run/github-runner-registered + if [ $succeeded -gt 0 ]; then + log "$succeeded runner(s) registered and started successfully" + touch $V-registered + else + log_error "No runners registered successfully" + fi - if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if [ -f $V-watchdog.pid ]; then + WATCHDOG_PID=$(cat $V-watchdog.pid) kill $WATCHDOG_PID 2>/dev/null || true - rm -f /var/run/github-runner-watchdog.pid + rm -f $V-watchdog.pid fi - touch /var/run/github-runner-started + touch $V-started chmod o+x $homedir for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done + ''' # --- # name: test_build_user_data_with_cloudwatch @@ -1459,7 +1573,7 @@ break fi done - + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do if [ -d "$dir" ]; then @@ -1480,11 +1594,17 @@ fi export homedir - cat > /usr/local/bin/runner-common.sh << EOSF + B=/usr/local/bin + V=/var/run/github-runner + + cat > $B/runner-common.sh << EOSF homedir="$homedir" debug="$debug" export homedir debug + EOSF + + cat >> $B/runner-common.sh << 'EOSF' log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } @@ -1538,7 +1658,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug mode: Sleeping for 600 seconds before shutdown to allow debugging..." + log "Debug: Sleeping 600s before shutdown..." local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" @@ -1591,8 +1711,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing runner dependencies..." - sudo ./bin/installdependencies.sh >/dev/null 2>&1 || log "Warning: Some dependencies may have failed to install" + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing required packages manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi + fi fi echo "$token" > .runner-token @@ -1623,12 +1759,14 @@ return 0 } EOSF - chmod +x /usr/local/bin/runner-common.sh - source /usr/local/bin/runner-common.sh + + chmod +x $B/runner-common.sh + source $B/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR + trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM REGISTRATION_TIMEOUT="300" if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then @@ -1636,19 +1774,22 @@ REGISTRATION_TIMEOUT=300 fi logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-active ( log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" sleep $REGISTRATION_TIMEOUT - if [ ! -f /var/run/github-runner-registered ]; then + if [ ! -f $V-registered ]; then log "Watchdog: Registration marker not found after timeout" - terminate_instance "Runner failed to register within $REGISTRATION_TIMEOUT seconds" + touch $V-watchdog-terminate + kill -TERM $$ 2>/dev/null || true else log "Watchdog: Registration marker found, exiting normally" fi + rm -f $V-watchdog-active ) & REGISTRATION_WATCHDOG_PID=$! log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > /var/run/github-runner-watchdog.pid + echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid @@ -1682,16 +1823,20 @@ else log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" fi + P='"file_path":' + G=',"log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$homedir" cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {"file_path":"/var/log/runner-setup.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-setup","timezone":"UTC"}, - {"file_path":"/var/log/runner-debug.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-debug","timezone":"UTC"}, - {"file_path":"/tmp/job-started-hook.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/job-started","timezone":"UTC"}, - {"file_path":"/tmp/job-completed-hook.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/job-completed","timezone":"UTC"}, - {"file_path":"/tmp/termination-check.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/termination","timezone":"UTC"}, - {"file_path":"/tmp/runner-*-config.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-config","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Runner_**.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/runner-diag","timezone":"UTC"}, - {"file_path":"/home/ec2-user/_diag/Worker_**.log","log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/worker-diag","timezone":"UTC"} + {$P"/var/log/runner-setup.log"${G}runner-setup$Z, + {$P"/var/log/runner-debug.log"${G}runner-debug$Z, + {$P"/tmp/job-started-hook.log"${G}job-started$Z, + {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, + {$P"/tmp/termination-check.log"${G}termination$Z, + {$P"/tmp/runner-*-config.log"${G}runner-config$Z, + {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, + {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z ]}}}} EOF /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s @@ -1741,54 +1886,50 @@ fi log "Downloaded runner binary" - cat > /usr/local/bin/job-started-hook.sh << 'EOFS' + cat > $B/job-started-hook.sh << EOFS #!/bin/bash exec >> /tmp/job-started-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job started: ${GITHUB_JOB}" + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" mkdir -p $V-jobs - echo '{\"status\":\"running\",\"runner\":\"'$RUNNER_IDX'\"}' > $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity $V-has-run-job EOFS - cat > /usr/local/bin/job-completed-hook.sh << 'EOFC' + cat > $B/job-completed-hook.sh << EOFC #!/bin/bash exec >> /tmp/job-completed-hook.log 2>&1 - V="/var/run/github-runner" - RUNNER_IDX="${RUNNER_INDEX:-0}" - echo "[$(date)] Runner-$RUNNER_IDX: Job completed: ${GITHUB_JOB}" - rm -f $V-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$RUNNER_IDX.job + I="\${RUNNER_INDEX:-0}" + echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" + rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job touch $V-last-activity EOFC - cat > /usr/local/bin/check-runner-termination.sh << EOFT + cat > $B/check-runner-termination.sh << EOFT #!/bin/bash exec >> /tmp/termination-check.log 2>&1 - source /usr/local/bin/runner-common.sh - V="/var/run/github-runner" - A="\\$V-last-activity" - J="\\$V-jobs" - H="\\$V-has-run-job" - [ ! -f "\\$A" ] && touch "\\$A" - L=\\$(stat -c %Y "\\$A" 2>/dev/null || echo 0) - N=\\$(date +%s) - I=\\$((N-L)) - [ -f "\\$H" ] && G=\\${RUNNER_GRACE_PERIOD:-60} || G=\\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\\$(grep -l '\"status\":\"running\"' \\$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \\$R -eq 0 ] && [ \\$I -gt \\$G ]; then - log "TERMINATING: idle \\$I > grace \\$G" + source $B/runner-common.sh + A="$V-last-activity" + J="$V-jobs" + H="$V-has-run-job" + [ ! -f "\$A" ] && touch "\$A" + L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) + N=\$(date +%s) + I=\$((N-L)) + [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} + R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) + if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then + log "TERMINATING: idle \$I > grace \$G" deregister_all_runners flush_cloudwatch_logs debug_sleep_and_shutdown else - [ \\$R -gt 0 ] && log "\\$R job(s) running" || log "Idle \\$I/\\$G sec" + [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" fi EOFT - chmod +x /usr/local/bin/job-started-hook.sh /usr/local/bin/job-completed-hook.sh /usr/local/bin/check-runner-termination.sh + chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - V="/var/run/github-runner" mkdir -p $V-jobs touch $V-last-activity @@ -1800,7 +1941,7 @@ Type=oneshot Environment="RUNNER_GRACE_PERIOD=61" Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=/usr/local/bin/check-runner-termination.sh + ExecStart=$B/check-runner-termination.sh EOF cat > /etc/systemd/system/runner-termination-check.timer << EOF @@ -1852,6 +1993,7 @@ continue fi ( + trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" echo $? > /tmp/runner-$i-status ) & @@ -1861,6 +2003,7 @@ log "Waiting for all runner configurations to complete..." failed=0 + succeeded=0 for i in ${!pids[@]}; do wait ${pids[$i]} if [ -f /tmp/runner-$i-status ]; then @@ -1868,28 +2011,37 @@ rm -f /tmp/runner-$i-status if [ "$status" != "0" ]; then log_error "Runner $i configuration failed" - failed=1 + failed=$((failed + 1)) + else + succeeded=$((succeeded + 1)) fi fi done - if [ $failed -ne 0 ]; then - terminate_instance "One or more runners failed to register" + if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then + terminate_instance "All runners failed to register" + elif [ $failed -gt 0 ]; then + log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." fi - log "All runners registered and started" - touch /var/run/github-runner-registered + if [ $succeeded -gt 0 ]; then + log "$succeeded runner(s) registered and started successfully" + touch $V-registered + else + log_error "No runners registered successfully" + fi - if [ -f /var/run/github-runner-watchdog.pid ]; then - WATCHDOG_PID=$(cat /var/run/github-runner-watchdog.pid) + if [ -f $V-watchdog.pid ]; then + WATCHDOG_PID=$(cat $V-watchdog.pid) kill $WATCHDOG_PID 2>/dev/null || true - rm -f /var/run/github-runner-watchdog.pid + rm -f $V-watchdog.pid fi - touch /var/run/github-runner-started + touch $V-started chmod o+x $homedir for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" done + ''' # --- From 7d04a6e5babcfa91bcf9e1c7968708549bb7484c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 13:29:24 -0400 Subject: [PATCH 31/69] mv `shared-functions.sh{.templ,}` --- pyproject.toml | 2 +- src/ec2_gha/start.py | 6 +++--- .../{shared-functions.sh.templ => shared-functions.sh} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename src/ec2_gha/templates/{shared-functions.sh.templ => shared-functions.sh} (100%) diff --git a/pyproject.toml b/pyproject.toml index b224296..787f117 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -ec2_gha = ["*.templ", "templates/*.templ"] +ec2_gha = ["*.templ", "templates/*.templ", "templates/*.sh"] [tool.pytest.ini_options] markers = ["slow: marks test as slow"] diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 1a1279a..854fa85 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -227,9 +227,9 @@ def _build_user_data(self, **kwargs) -> str: kwargs['log_prefix_job_started'] = LOG_PREFIX_JOB_STARTED kwargs['log_prefix_job_completed'] = LOG_PREFIX_JOB_COMPLETED - # Load shared functions template - don't render it, just include as-is - shared_functions_template = importlib.resources.files("ec2_gha").joinpath("templates/shared-functions.sh.templ") - with shared_functions_template.open() as f: + # Load shared functions script - not a template, just include as-is + shared_functions_file = importlib.resources.files("ec2_gha").joinpath("templates/shared-functions.sh") + with shared_functions_file.open() as f: shared_functions_content = f.read() # Strip the shebang line from shared functions since it will be embedded diff --git a/src/ec2_gha/templates/shared-functions.sh.templ b/src/ec2_gha/templates/shared-functions.sh similarity index 100% rename from src/ec2_gha/templates/shared-functions.sh.templ rename to src/ec2_gha/templates/shared-functions.sh From 9104e9743f86cbe016ad3f4bc71d98af98865a50 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 13:54:18 -0400 Subject: [PATCH 32/69] demo-archs fixes --- .github/workflows/demo-archs.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/demo-archs.yml b/.github/workflows/demo-archs.yml index 70c972b..fcd97aa 100644 --- a/.github/workflows/demo-archs.yml +++ b/.github/workflows/demo-archs.yml @@ -21,7 +21,7 @@ permissions: jobs: # Launch EC2 runners for each OS/architecture combination ubuntu-amd64: - name: Launch Ubuntu AMD64 + name: 🚀 Ubuntu AMD uses: ./.github/workflows/runner.yml with: ec2_instance_type: t3.medium @@ -30,7 +30,7 @@ jobs: secrets: inherit ubuntu-arm64: - name: Launch Ubuntu ARM64 + name: 🚀 Ubuntu ARM uses: ./.github/workflows/runner.yml with: ec2_instance_type: t4g.medium @@ -39,7 +39,7 @@ jobs: secrets: inherit debian-amd64: - name: Launch Debian AMD64 + name: 🚀 Debian AMD uses: ./.github/workflows/runner.yml with: ec2_instance_type: t3.large @@ -48,7 +48,7 @@ jobs: secrets: inherit debian-arm64: - name: Launch Debian ARM64 + name: 🚀 Debian ARM uses: ./.github/workflows/runner.yml with: ec2_instance_type: t4g.large @@ -57,7 +57,7 @@ jobs: secrets: inherit al2023-amd64: - name: Launch AL2023 AMD64 + name: 🚀 AL2023 AMD uses: ./.github/workflows/runner.yml with: ec2_instance_type: t3.small @@ -66,7 +66,7 @@ jobs: secrets: inherit al2023-arm64: - name: Launch AL2023 ARM64 + name: 🚀 AL2023 ARM uses: ./.github/workflows/runner.yml with: ec2_instance_type: t4g.small @@ -77,7 +77,7 @@ jobs: # Run test jobs on the launched instances using a matrix test-matrix: needs: [ubuntu-amd64, ubuntu-arm64, debian-amd64, debian-arm64, al2023-amd64, al2023-arm64] - name: Test ${{ matrix.os }} ${{ matrix.arch }} + name: 🔬 ${{ matrix.os }} ${{ matrix.arch }} strategy: matrix: include: @@ -135,17 +135,21 @@ jobs: cat /etc/os-release | head -5 echo "" echo "=== AWS Instance Details ===" - echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" - echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" - echo "Region: $(curl -s http://169.254.169.254/latest/meta-data/placement/region)" - echo "AZ: $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)" + # Get IMDSv2 token for metadata access + TOKEN=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" -s http://169.254.169.254/latest/api/token) + echo "Instance ID: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id)" + echo "Instance type: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-type)" + echo "Region: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/placement/region)" + echo "AZ: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/placement/availability-zone)" - name: Verify OS and architecture run: | echo "=== Verification ===" ACTUAL_ARCH=$(uname -m) OS_NAME=$(grep '^NAME=' /etc/os-release | cut -d'"' -f2) - ACTUAL_INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) + # Get IMDSv2 token for metadata access + TOKEN=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" -s http://169.254.169.254/latest/api/token) + ACTUAL_INSTANCE_TYPE=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-type) # Verify architecture if [[ "$ACTUAL_ARCH" == "${{ matrix.expected_arch }}" ]]; then From 12f10880303c50054cf1e5a97659395bbe362cac Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 14:16:34 -0400 Subject: [PATCH 33/69] improve runner labels, factor `template_vars` --- src/ec2_gha/start.py | 117 +++++++++++++-------- src/ec2_gha/templates/user-script.sh.templ | 11 +- tests/__snapshots__/test_start.ambr | 40 +++---- 3 files changed, 95 insertions(+), 73 deletions(-) diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 854fa85..00b695c 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -94,6 +94,65 @@ class StartAWS(CreateCloudInstance): tags: list[dict[str, str]] = field(default_factory=list) userdata: str = "" + def _get_template_vars(self, idx: int = None) -> dict: + """Build template variables for instance naming. + + Parameters + ---------- + idx : int | None + Instance index for multi-instance launches + + Returns + ------- + dict + Dictionary of template variables for string substitution + """ + from os import environ + import re + + template_vars = {} + + # Get repository name (just the basename) + if environ.get("GITHUB_REPOSITORY"): + template_vars["repo"] = environ["GITHUB_REPOSITORY"].split("/")[-1] + else: + template_vars["repo"] = "unknown" + + # Get workflow full name (e.g., "Test pip install") + template_vars["workflow"] = environ.get("GITHUB_WORKFLOW", "unknown") + + # Get workflow filename stem and ref from GITHUB_WORKFLOW_REF + workflow_ref = environ.get("GITHUB_WORKFLOW_REF", "") + if workflow_ref: + # Extract filename and ref from path like "owner/repo/.github/workflows/test.yml@ref" + m = re.search(r'/(?P[^/@]+)\.(yml|yaml)@(?P[^@]+)$', workflow_ref) + if m: + # Get the workflow filename stem (e.g., "install" from "install.yaml") + template_vars["name"] = m['name'] + + # Clean up the ref - remove "refs/heads/" or "refs/tags/" prefix + ref = m['ref'] + if ref.startswith('refs/heads/'): + ref = ref[11:] + elif ref.startswith('refs/tags/'): + ref = ref[10:] + template_vars["ref"] = ref + else: + template_vars["name"] = "unknown" + template_vars["ref"] = "unknown" + else: + template_vars["name"] = "unknown" + template_vars["ref"] = "unknown" + + # Get run number + template_vars["run_number"] = environ.get("GITHUB_RUN_NUMBER", "unknown") + + # Add instance index if provided (for multi-instance launches) + if idx is not None: + template_vars["idx"] = str(idx) + + return template_vars + def _build_aws_params(self, user_data_params: dict, idx: int = None) -> dict: """Build the parameters for the AWS API call. @@ -127,52 +186,11 @@ def _build_aws_params(self, user_data_params: dict, idx: int = None) -> dict: # Add default tags if not already present default_tags = [] existing_keys = {tag["Key"] for tag in self.tags} - import os # Add Name tag if not provided if "Name" not in existing_keys: - # Build template variables - template_vars = {} - - # Get repository name (just the basename) - if environ.get("GITHUB_REPOSITORY"): - template_vars["repo"] = environ["GITHUB_REPOSITORY"].split("/")[-1] - else: - template_vars["repo"] = "unknown" - - # Get workflow full name (e.g., "Test pip install") - template_vars["workflow"] = environ.get("GITHUB_WORKFLOW", "unknown") - - # Get workflow filename stem and ref from GITHUB_WORKFLOW_REF - workflow_ref = environ.get("GITHUB_WORKFLOW_REF", "") - if workflow_ref: - import re - # Extract filename and ref from path like "owner/repo/.github/workflows/test.yml@ref" - m = re.search(r'/(?P[^/@]+)\.(yml|yaml)@(?P[^@]+)$', workflow_ref) - if m: - # Get the workflow filename stem (e.g., "install" from "install.yaml") - template_vars["name"] = m['name'] - - # Clean up the ref - remove "refs/heads/" or "refs/tags/" prefix - ref = m['ref'] - if ref.startswith('refs/heads/'): - ref = ref[11:] - elif ref.startswith('refs/tags/'): - ref = ref[10:] - template_vars["ref"] = ref - else: - template_vars["name"] = "unknown" - template_vars["ref"] = "unknown" - else: - template_vars["name"] = "unknown" - template_vars["ref"] = "unknown" - - # Get run number - template_vars["run_number"] = environ.get("GITHUB_RUN_NUMBER", "unknown") - - # Add instance index if provided (for multi-instance launches) - if idx is not None: - template_vars["idx"] = str(idx) + # Get template variables + template_vars = self._get_template_vars(idx) # Apply the instance name template from string import Template @@ -227,6 +245,9 @@ def _build_user_data(self, **kwargs) -> str: kwargs['log_prefix_job_started'] = LOG_PREFIX_JOB_STARTED kwargs['log_prefix_job_completed'] = LOG_PREFIX_JOB_COMPLETED + # Ensure instance_name has a default value + kwargs.setdefault('instance_name', '') + # Load shared functions script - not a template, just include as-is shared_functions_file = importlib.resources.files("ec2_gha").joinpath("templates/shared-functions.sh") with shared_functions_file.open() as f: @@ -352,6 +373,15 @@ def create_instances(self) -> dict[str, str]: runner_tokens = " ".join(config["token"] for config in runner_configs) runner_labels = "|".join(config["labels"] for config in runner_configs) + # Generate instance name using template variables + if self.instance_name: + from string import Template + template_vars = self._get_template_vars(idx) + name_template = Template(self.instance_name) + instance_name_value = name_template.safe_substitute(**template_vars) + else: + instance_name_value = "" + user_data_params = { "cloudwatch_logs_group": self.cloudwatch_logs_group, "debug": self.debug, @@ -359,6 +389,7 @@ def create_instances(self) -> dict[str, str]: "github_run_id": environ.get("GITHUB_RUN_ID", ""), "github_run_number": environ.get("GITHUB_RUN_NUMBER", ""), "homedir": self.home_dir, + "instance_name": instance_name_value, # Add the generated instance name "max_instance_lifetime": self.max_instance_lifetime, "repo": self.repo, "runner_grace_period": self.runner_grace_period, diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 85ed2f0..c9889cf 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -289,13 +289,12 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer # Build metadata labels (these will be added to the runner labels) -METADATA_LABELS=",instance-id:$${INSTANCE_ID},instance-type:$${INSTANCE_TYPE}" -if [ -n "$github_workflow" ]; then - WORKFLOW_LABEL=$$(echo "$github_workflow" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="$${METADATA_LABELS},workflow:$${WORKFLOW_LABEL}" +METADATA_LABELS=",$${INSTANCE_ID},$${INSTANCE_TYPE}" +# Add instance name as a label if provided +if [ -n "$instance_name" ]; then + INSTANCE_NAME_LABEL=$$(echo "$instance_name" | tr ' /' '-' | tr -cd '[:alnum:]-_#') + METADATA_LABELS="$${METADATA_LABELS},$${INSTANCE_NAME_LABEL}" fi -[ -n "$github_run_id" ] && METADATA_LABELS="$${METADATA_LABELS},run-id:$github_run_id" -[ -n "$github_run_number" ] && METADATA_LABELS="$${METADATA_LABELS},run-number:$github_run_number" log "Setting up $$RUNNERS_PER_INSTANCE runner(s)" diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index 71da414..d738a64 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -445,13 +445,11 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" - if [ -n "CI" ]; then - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" + if [ -n "" ]; then + INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') + METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" fi - [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - [ -n "1" ] && METADATA_LABELS="${METADATA_LABELS},run-number:1" log "Setting up $RUNNERS_PER_INSTANCE runner(s)" @@ -974,13 +972,11 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" - if [ -n "CI" ]; then - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" + if [ -n "" ]; then + INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') + METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" fi - [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - [ -n "42" ] && METADATA_LABELS="${METADATA_LABELS},run-number:42" log "Setting up $RUNNERS_PER_INSTANCE runner(s)" @@ -1467,13 +1463,11 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" - if [ -n "CI" ]; then - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" + if [ -n "" ]; then + INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') + METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" fi - [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - [ -n "42" ] && METADATA_LABELS="${METADATA_LABELS},run-number:42" log "Setting up $RUNNERS_PER_INSTANCE runner(s)" @@ -1959,13 +1953,11 @@ systemctl enable runner-termination-check.timer systemctl start runner-termination-check.timer - METADATA_LABELS=",instance-id:${INSTANCE_ID},instance-type:${INSTANCE_TYPE}" - if [ -n "CI" ]; then - WORKFLOW_LABEL=$(echo "CI" | tr ' /' '-' | tr -cd '[:alnum:]-_') - METADATA_LABELS="${METADATA_LABELS},workflow:${WORKFLOW_LABEL}" + METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" + if [ -n "" ]; then + INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') + METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" fi - [ -n "16725250800" ] && METADATA_LABELS="${METADATA_LABELS},run-id:16725250800" - [ -n "42" ] && METADATA_LABELS="${METADATA_LABELS},run-number:42" log "Setting up $RUNNERS_PER_INSTANCE runner(s)" From b23db5f465c9688a4550711bf773c26d85bf428d Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 14:56:11 -0400 Subject: [PATCH 34/69] `demo-cpu-sweep.yml` --- .github/workflows/demo-archs.yml | 2 +- .github/workflows/demo-cpu-sweep.yml | 43 +++++++++++++++++++++++ .github/workflows/demo-dbg-minimal.yml | 32 ++++++++++++++--- .github/workflows/demo-gpu-job-seq.yml | 2 +- .github/workflows/demo-job-seq.yml | 4 +-- .github/workflows/demo-matrix-wide.yml | 2 +- .github/workflows/demo-multi-instance.yml | 4 +-- .github/workflows/demo-multi-job.yml | 4 +-- .github/workflows/demo-multi-runner.yml | 2 +- .github/workflows/runner.yml | 2 +- 10 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/demo-cpu-sweep.yml diff --git a/.github/workflows/demo-archs.yml b/.github/workflows/demo-archs.yml index fcd97aa..1a36776 100644 --- a/.github/workflows/demo-archs.yml +++ b/.github/workflows/demo-archs.yml @@ -16,7 +16,7 @@ on: permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: # Launch EC2 runners for each OS/architecture combination diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml new file mode 100644 index 0000000..7cd5dc9 --- /dev/null +++ b/.github/workflows/demo-cpu-sweep.yml @@ -0,0 +1,43 @@ +name: Demo – OS/Architecture sweep (CPU nodes) +on: + workflow_dispatch: + inputs: + sleep: + description: "Sleep duration in seconds for workload simulation" + required: false + type: string + default: "10" + workflow_call: # Can be called from other workflows + inputs: + sleep: + required: false + type: string + default: "10" +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout +jobs: + os-arch-matrix: + name: ${{ matrix.os }} ${{ matrix.version }} ${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - { "os": Ubuntu, "version": "22.04", "arch": x86_64, "ami": ami-021589336d307b577, "instance_type": t3.medium } + - { "os": Ubuntu, "version": "22.04", "arch": arm64, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } + - { "os": Ubuntu, "version": "24.04", "arch": x86_64, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } + - { "os": Ubuntu, "version": "24.04", "arch": arm64, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } + - { "os": Debian, "version": "11", "arch": x86_64, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } + - { "os": Debian, "version": "11", "arch": arm64, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } + - { "os": Debian, "version": "12", "arch": x86_64, "ami": ami-05b50089e01b13194, "instance_type": t3.large } + - { "os": Debian, "version": "12", "arch": arm64, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } + - { "os": AL2, "version": "2", "arch": x86_64, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } + - { "os": AL2, "version": "2", "arch": arm64, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } + - { "os": AL2023, "version": "2023", "arch": x86_64, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } + - { "os": AL2023, "version": "2023", "arch": arm64, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } + uses: ./.github/workflows/demo-dbg-minimal.yml + with: + type: ${{ matrix.instance_type }} + ami: ${{ matrix.ami }} + sleep: ${{ inputs.sleep }} + secrets: inherit diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml index ccfac89..c5de293 100644 --- a/.github/workflows/demo-dbg-minimal.yml +++ b/.github/workflows/demo-dbg-minimal.yml @@ -42,10 +42,32 @@ on: required: false type: boolean default: true + workflow_call: # Called by demo-cpu-sweep, more for the "minimal", less for the "dbg" + inputs: + type: + description: "EC2 instance type" + required: false + type: string + default: "t3.large" + ami: + description: "AMI ID to use" + required: false + type: string + default: "ami-0e86e20dae9224db8" # Ubuntu 24.04 LTS x86_64 (us-east-1) + sleep: + description: "Sleep duration in seconds" + required: false + type: string + default: "10" + debug: + description: "Enable debug mode" + required: false + type: boolean + default: false permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: launch: @@ -56,10 +78,10 @@ jobs: ec2_image_id: ${{ inputs.ami }} debug: ${{ inputs.debug }} instance_name: "debug/$name#$run_number" - runner_registration_timeout: ${{ inputs.registration_timeout }} - runner_grace_period: ${{ inputs.grace_period }} - runner_initial_grace_period: ${{ inputs.initial_grace_period }} - max_instance_lifetime: ${{ inputs.max_instance_lifetime }} + runner_registration_timeout: ${{ inputs.registration_timeout || '300' }} + runner_grace_period: ${{ inputs.grace_period || '60' }} + runner_initial_grace_period: ${{ inputs.initial_grace_period || '120' }} + max_instance_lifetime: ${{ inputs.max_instance_lifetime || '60' }} secrets: inherit test: diff --git a/.github/workflows/demo-gpu-job-seq.yml b/.github/workflows/demo-gpu-job-seq.yml index 301df5a..489cee4 100644 --- a/.github/workflows/demo-gpu-job-seq.yml +++ b/.github/workflows/demo-gpu-job-seq.yml @@ -23,7 +23,7 @@ on: permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: ec2: diff --git a/.github/workflows/demo-job-seq.yml b/.github/workflows/demo-job-seq.yml index 6bb51b6..0894991 100644 --- a/.github/workflows/demo-job-seq.yml +++ b/.github/workflows/demo-job-seq.yml @@ -16,8 +16,8 @@ on: default: 0 permissions: - id-token: write - contents: read + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: ec2: diff --git a/.github/workflows/demo-matrix-wide.yml b/.github/workflows/demo-matrix-wide.yml index e7e3949..9681426 100644 --- a/.github/workflows/demo-matrix-wide.yml +++ b/.github/workflows/demo-matrix-wide.yml @@ -4,7 +4,7 @@ on: workflow_call: # Tested by `demos.yml` permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: ec2: uses: ./.github/workflows/runner.yml diff --git a/.github/workflows/demo-multi-instance.yml b/.github/workflows/demo-multi-instance.yml index a2b97f2..da0ac6a 100644 --- a/.github/workflows/demo-multi-instance.yml +++ b/.github/workflows/demo-multi-instance.yml @@ -16,7 +16,7 @@ on: permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: ec2: @@ -58,4 +58,4 @@ jobs: - name: Verify parallelism run: | echo "Completed at: $(date '+%Y-%m-%d %H:%M:%S')" - echo "This job ran in parallel with other matrix jobs" \ No newline at end of file + echo "This job ran in parallel with other matrix jobs" diff --git a/.github/workflows/demo-multi-job.yml b/.github/workflows/demo-multi-job.yml index d1e0ebb..484c33a 100644 --- a/.github/workflows/demo-multi-job.yml +++ b/.github/workflows/demo-multi-job.yml @@ -5,7 +5,7 @@ on: permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: ec2: @@ -87,4 +87,4 @@ jobs: echo "Test report:" cat test-report/test-report.txt echo "" - echo "Both jobs completed successfully using separate EC2 instances!" \ No newline at end of file + echo "Both jobs completed successfully using separate EC2 instances!" diff --git a/.github/workflows/demo-multi-runner.yml b/.github/workflows/demo-multi-runner.yml index c6d2c17..ac5618b 100644 --- a/.github/workflows/demo-multi-runner.yml +++ b/.github/workflows/demo-multi-runner.yml @@ -25,7 +25,7 @@ on: permissions: id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout + contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. jobs: ec2: diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index fd4c17e..fa06ca9 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -131,7 +131,7 @@ on: value: ${{ jobs.launch.outputs.mapping }} permissions: - id-token: write # Required for AWS OIDC + id-token: write # Required for AWS OIDC jobs: launch: From cf175b5e7890985146d2e82e89fc5dd09c6becba Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 16:17:29 -0400 Subject: [PATCH 35/69] `name` param --- .github/workflows/demo-cpu-sweep.yml | 24 ++++++++++++------------ .github/workflows/demo-dbg-minimal.yml | 4 +++- .github/workflows/runner.yml | 6 +++++- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml index 7cd5dc9..7637bcd 100644 --- a/.github/workflows/demo-cpu-sweep.yml +++ b/.github/workflows/demo-cpu-sweep.yml @@ -23,18 +23,18 @@ jobs: fail-fast: false matrix: include: - - { "os": Ubuntu, "version": "22.04", "arch": x86_64, "ami": ami-021589336d307b577, "instance_type": t3.medium } - - { "os": Ubuntu, "version": "22.04", "arch": arm64, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } - - { "os": Ubuntu, "version": "24.04", "arch": x86_64, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } - - { "os": Ubuntu, "version": "24.04", "arch": arm64, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } - - { "os": Debian, "version": "11", "arch": x86_64, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } - - { "os": Debian, "version": "11", "arch": arm64, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } - - { "os": Debian, "version": "12", "arch": x86_64, "ami": ami-05b50089e01b13194, "instance_type": t3.large } - - { "os": Debian, "version": "12", "arch": arm64, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } - - { "os": AL2, "version": "2", "arch": x86_64, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } - - { "os": AL2, "version": "2", "arch": arm64, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } - - { "os": AL2023, "version": "2023", "arch": x86_64, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } - - { "os": AL2023, "version": "2023", "arch": arm64, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } + - { "os": Ubuntu, "version": "22.04", "arch": x86, "ami": ami-021589336d307b577, "instance_type": t3.medium } + - { "os": Ubuntu, "version": "22.04", "arch": ARM, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } + - { "os": Ubuntu, "version": "24.04", "arch": x86, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } + - { "os": Ubuntu, "version": "24.04", "arch": ARM, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } + - { "os": Debian, "version": "11", "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } + - { "os": Debian, "version": "11", "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } + - { "os": Debian, "version": "12", "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } + - { "os": Debian, "version": "12", "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } + - { "os": AL2, "version": "2", "arch": x86, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } + - { "os": AL2, "version": "2", "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } + - { "os": AL2023, "version": "2023", "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } + - { "os": AL2023, "version": "2023", "arch": ARM, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } uses: ./.github/workflows/demo-dbg-minimal.yml with: type: ${{ matrix.instance_type }} diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml index c5de293..f48818a 100644 --- a/.github/workflows/demo-dbg-minimal.yml +++ b/.github/workflows/demo-dbg-minimal.yml @@ -71,13 +71,15 @@ permissions: jobs: launch: - name: Launch Debug Instance + name: Launch ${{ inputs.type }} uses: ./.github/workflows/runner.yml with: + name: '🚀' ec2_instance_type: ${{ inputs.type }} ec2_image_id: ${{ inputs.ami }} debug: ${{ inputs.debug }} instance_name: "debug/$name#$run_number" + # `workflow_dispatch has higher defaults for these; revert to original defaults for `workflow_call` runner_registration_timeout: ${{ inputs.registration_timeout || '300' }} runner_grace_period: ${{ inputs.grace_period || '60' }} runner_initial_grace_period: ${{ inputs.initial_grace_period || '120' }} diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index fa06ca9..35dcd3b 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -94,6 +94,10 @@ on: description: "Maximum instance lifetime in minutes before automatic shutdown (falls back to vars.MAX_INSTANCE_LIFETIME, then 360 = 6 hours)" required: false type: string + name: + description: "Name for the launch job" + required: false + type: string runner_grace_period: description: "Grace period in seconds before terminating instance after last job completes (falls back to vars.RUNNER_GRACE_PERIOD, then 60)" required: false @@ -135,7 +139,7 @@ permissions: jobs: launch: - name: Launch ${{ inputs.ec2_instance_type || vars.EC2_INSTANCE_TYPE }} + name: ${{ inputs.name || format('Launch {0}', inputs.ec2_instance_type || vars.EC2_INSTANCE_TYPE) }} runs-on: ubuntu-latest outputs: id: ${{ steps.aws-start.outputs.label }} From f5a7f8120a4a07de8c9384f52da540f5691344dc Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 19:00:06 -0400 Subject: [PATCH 36/69] demos refactor --- .github/workflows/demo-archs.yml | 268 ------------------ .github/workflows/demo-gpu-job-seq.yml | 137 --------- .github/workflows/demo-gpu-sweep.yml | 102 +++++++ ...ti-instance.yml => demo-instances-mtx.yml} | 2 +- .github/workflows/demo-job-seq.yml | 76 ----- ...demo-multi-job.yml => demo-jobs-split.yml} | 4 +- .github/workflows/demo-matrix-wide.yml | 37 --- ...-multi-runner.yml => demo-runners-mtx.yml} | 2 +- .github/workflows/demos.yml | 33 ++- .github/workflows/runner.yml | 8 +- action.yml | 2 +- src/ec2_gha/start.py | 2 +- 12 files changed, 131 insertions(+), 542 deletions(-) delete mode 100644 .github/workflows/demo-archs.yml delete mode 100644 .github/workflows/demo-gpu-job-seq.yml create mode 100644 .github/workflows/demo-gpu-sweep.yml rename .github/workflows/{demo-multi-instance.yml => demo-instances-mtx.yml} (97%) delete mode 100644 .github/workflows/demo-job-seq.yml rename .github/workflows/{demo-multi-job.yml => demo-jobs-split.yml} (95%) delete mode 100644 .github/workflows/demo-matrix-wide.yml rename .github/workflows/{demo-multi-runner.yml => demo-runners-mtx.yml} (98%) diff --git a/.github/workflows/demo-archs.yml b/.github/workflows/demo-archs.yml deleted file mode 100644 index 1a36776..0000000 --- a/.github/workflows/demo-archs.yml +++ /dev/null @@ -1,268 +0,0 @@ -name: Demo – OS and architecture matrix -on: - workflow_dispatch: - inputs: - sleep: - description: "Sleep duration in seconds for workload simulation" - required: false - type: string - default: "10" - workflow_call: # Tested by `demos.yml` - inputs: - sleep: - required: false - type: string - default: "10" - -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. - -jobs: - # Launch EC2 runners for each OS/architecture combination - ubuntu-amd64: - name: 🚀 Ubuntu AMD - uses: ./.github/workflows/runner.yml - with: - ec2_instance_type: t3.medium - ec2_image_id: ami-0ca5a2f40c2601df6 # Ubuntu 24.04 LTS x86_64 (us-east-1) - instance_name: "Ubuntu-AMD64/$name#$run_number" - secrets: inherit - - ubuntu-arm64: - name: 🚀 Ubuntu ARM - uses: ./.github/workflows/runner.yml - with: - ec2_instance_type: t4g.medium - ec2_image_id: ami-0aa307ed50ca3e58f # Ubuntu 24.04 LTS arm64 (us-east-1) - instance_name: "Ubuntu-ARM64/$name#$run_number" - secrets: inherit - - debian-amd64: - name: 🚀 Debian AMD - uses: ./.github/workflows/runner.yml - with: - ec2_instance_type: t3.large - ec2_image_id: ami-05b50089e01b13194 # Debian 12 x86_64 (us-east-1) - instance_name: "Debian-AMD64/$name#$run_number" - secrets: inherit - - debian-arm64: - name: 🚀 Debian ARM - uses: ./.github/workflows/runner.yml - with: - ec2_instance_type: t4g.large - ec2_image_id: ami-0505441d7e1514742 # Debian 12 arm64 (us-east-1) - instance_name: "Debian-ARM64/$name#$run_number" - secrets: inherit - - al2023-amd64: - name: 🚀 AL2023 AMD - uses: ./.github/workflows/runner.yml - with: - ec2_instance_type: t3.small - ec2_image_id: ami-00ca32bbc84273381 # Amazon Linux 2023 x86_64 (us-east-1) - instance_name: "AL2023-AMD64/$name#$run_number" - secrets: inherit - - al2023-arm64: - name: 🚀 AL2023 ARM - uses: ./.github/workflows/runner.yml - with: - ec2_instance_type: t4g.small - ec2_image_id: ami-0aa7db6294d00216f # Amazon Linux 2023 arm64 (us-east-1) - instance_name: "AL2023-ARM64/$name#$run_number" - secrets: inherit - - # Run test jobs on the launched instances using a matrix - test-matrix: - needs: [ubuntu-amd64, ubuntu-arm64, debian-amd64, debian-arm64, al2023-amd64, al2023-arm64] - name: 🔬 ${{ matrix.os }} ${{ matrix.arch }} - strategy: - matrix: - include: - - os: Ubuntu - arch: AMD64 - runner: ${{ needs.ubuntu-amd64.outputs.id }} - expected_arch: x86_64 - expected_os_pattern: "Ubuntu" - instance_type: t3.medium - - os: Ubuntu - arch: ARM64 - runner: ${{ needs.ubuntu-arm64.outputs.id }} - expected_arch: aarch64 - expected_os_pattern: "Ubuntu" - instance_type: t4g.medium - - os: Debian - arch: AMD64 - runner: ${{ needs.debian-amd64.outputs.id }} - expected_arch: x86_64 - expected_os_pattern: "Debian" - instance_type: t3.large - - os: Debian - arch: ARM64 - runner: ${{ needs.debian-arm64.outputs.id }} - expected_arch: aarch64 - expected_os_pattern: "Debian" - instance_type: t4g.large - - os: AL2023 - arch: AMD64 - runner: ${{ needs.al2023-amd64.outputs.id }} - expected_arch: x86_64 - expected_os_pattern: "Amazon Linux" - instance_type: t3.small - - os: AL2023 - arch: ARM64 - runner: ${{ needs.al2023-arm64.outputs.id }} - expected_arch: aarch64 - expected_os_pattern: "Amazon Linux" - instance_type: t4g.small - fail-fast: false - runs-on: ${{ matrix.runner }} - steps: - - name: System information - run: | - echo "=== System Information ===" - echo "OS: ${{ matrix.os }}" - echo "Architecture: ${{ matrix.arch }}" - echo "Expected instance type: ${{ matrix.instance_type }}" - echo "" - echo "=== Actual System Details ===" - echo "Hostname: $(hostname)" - echo "Kernel: $(uname -r)" - echo "Architecture: $(uname -m)" - echo "OS Release:" - cat /etc/os-release | head -5 - echo "" - echo "=== AWS Instance Details ===" - # Get IMDSv2 token for metadata access - TOKEN=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" -s http://169.254.169.254/latest/api/token) - echo "Instance ID: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-id)" - echo "Instance type: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-type)" - echo "Region: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/placement/region)" - echo "AZ: $(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/placement/availability-zone)" - - - name: Verify OS and architecture - run: | - echo "=== Verification ===" - ACTUAL_ARCH=$(uname -m) - OS_NAME=$(grep '^NAME=' /etc/os-release | cut -d'"' -f2) - # Get IMDSv2 token for metadata access - TOKEN=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" -s http://169.254.169.254/latest/api/token) - ACTUAL_INSTANCE_TYPE=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/instance-type) - - # Verify architecture - if [[ "$ACTUAL_ARCH" == "${{ matrix.expected_arch }}" ]]; then - echo "✓ Architecture verified: $ACTUAL_ARCH matches expected ${{ matrix.expected_arch }}" - else - echo "✗ Architecture mismatch: got $ACTUAL_ARCH, expected ${{ matrix.expected_arch }}" - exit 1 - fi - - # Verify OS - if echo "$OS_NAME" | grep -q "${{ matrix.expected_os_pattern }}"; then - echo "✓ OS verified: $OS_NAME matches expected pattern '${{ matrix.expected_os_pattern }}'" - else - echo "✗ OS mismatch: $OS_NAME does not match expected pattern '${{ matrix.expected_os_pattern }}'" - exit 1 - fi - - # Verify instance type - if [[ "$ACTUAL_INSTANCE_TYPE" == "${{ matrix.instance_type }}" ]]; then - echo "✓ Instance type verified: $ACTUAL_INSTANCE_TYPE" - else - echo "✗ Instance type mismatch: got $ACTUAL_INSTANCE_TYPE, expected ${{ matrix.instance_type }}" - exit 1 - fi - - - name: Performance characteristics - run: | - echo "=== Performance Characteristics ===" - echo "CPU cores: $(nproc)" - echo "CPU model: $(lscpu | grep 'Model name:' | cut -d: -f2 | xargs)" - echo "Memory: $(free -h | grep Mem | awk '{print $2}')" - echo "Disk: $(df -h / | tail -1 | awk '{print $2}')" - echo "" - echo "CPU details:" - lscpu | grep -E "Architecture:|CPU\(s\):|Thread\(s\) per core:|Core\(s\) per socket:|Socket\(s\):" - - - name: Simulate workload - run: | - DURATION=${{ inputs.sleep }} - echo "=== Workload Simulation ===" - echo "Starting workload at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" - echo "Running on ${{ matrix.os }} ${{ matrix.arch }} for ${DURATION} seconds..." - - # Architecture-specific workload - if [[ "${{ matrix.arch }}" == "ARM64" ]]; then - echo "Running ARM-optimized workload simulation..." - # Simulate ARM-specific operations - echo "ARM NEON/SVE capabilities check:" - grep -E "Features|flags" /proc/cpuinfo | head -2 || true - else - echo "Running x86-optimized workload simulation..." - # Simulate x86-specific operations - echo "x86 instruction set extensions:" - grep -E "flags" /proc/cpuinfo | head -1 | grep -o -E "sse|avx|avx2|avx512" | sort -u || true - fi - - sleep $DURATION - echo "Workload complete at: $(date '+%Y-%m-%d %H:%M:%S.%3N')" - - - name: Package manager test - run: | - echo "=== Package Manager Test ===" - if [[ "${{ matrix.os }}" == "Ubuntu" ]]; then - echo "Ubuntu uses APT package manager" - echo "APT version: $(apt-get --version | head -1)" - echo "Distribution: $(lsb_release -ds 2>/dev/null || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - elif [[ "${{ matrix.os }}" == "Debian" ]]; then - echo "Debian uses APT package manager" - echo "APT version: $(apt-get --version | head -1)" - echo "Distribution: $(cat /etc/debian_version)" - elif [[ "${{ matrix.os }}" == "AL2023" ]]; then - echo "Amazon Linux 2023 uses DNF/YUM package manager" - echo "DNF version: $(dnf --version | head -1)" - echo "Distribution: $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - fi - - # Show architecture-specific packages available - echo "" - echo "Architecture-specific package availability:" - if [[ "${{ matrix.arch }}" == "ARM64" ]]; then - echo "Checking for ARM64-specific packages..." - if [[ "${{ matrix.os }}" == "AL2023" ]]; then - rpm --eval '%{_arch}' - dnf repolist | head -5 - else - dpkg --print-architecture - dpkg --print-foreign-architectures || echo "No foreign architectures configured" - fi - else - echo "Checking for AMD64-specific packages..." - if [[ "${{ matrix.os }}" == "AL2023" ]]; then - rpm --eval '%{_arch}' - dnf repolist | head -5 - else - dpkg --print-architecture - dpkg --print-foreign-architectures || echo "No foreign architectures configured" - fi - fi - - - name: Summary - run: | - echo "=== Test Summary ===" - echo "✓ Successfully tested ${{ matrix.os }} on ${{ matrix.arch }} architecture" - echo "✓ Instance type: ${{ matrix.instance_type }}" - echo "✓ All verifications passed" - echo "" - echo "Matrix configuration demonstrated:" - echo "- OS diversity: Ubuntu 24.04, Debian 13, and Amazon Linux 2023" - echo "- Architecture diversity: AMD64 (x86_64) and ARM64 (aarch64)" - echo "- Instance type variety: t3.small, t3.medium, t3.large, t4g.small, t4g.medium, t4g.large" - echo "" - echo "This workflow shows ec2-gha's ability to:" - echo "1. Launch runners on different OS distributions" - echo "2. Support both x86 and ARM architectures" - echo "3. Use appropriate instance types for each architecture" - echo "4. Run parallel jobs across diverse infrastructure" diff --git a/.github/workflows/demo-gpu-job-seq.yml b/.github/workflows/demo-gpu-job-seq.yml deleted file mode 100644 index 489cee4..0000000 --- a/.github/workflows/demo-gpu-job-seq.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: Demo – simulated GPU workload with sequential jobs -on: - workflow_dispatch: - inputs: - sleep: - description: "Sleep for this many seconds at the end of each job (optional, for SSH debugging)" - required: false - type: number - default: 0 - ssh_pubkey: - description: "Add this SSH public key to instance's `~/.ssh/authorized_keys` (optional, for debugging)" - required: false - type: string - workflow_call: # Tested by `demos.yml` - inputs: - sleep: - required: false - type: number - default: 0 - ssh_pubkey: - required: false - type: string - -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. - -jobs: - ec2: - uses: ./.github/workflows/runner.yml - secrets: inherit - with: - ec2_instance_type: g4dn.xlarge - ec2_image_id: ami-00096836009b16a22 # Deep Learning OSS Nvidia Driver AMI GPU PyTorch - ssh_pubkey: ${{ inputs.ssh_pubkey }} - - # Job 1: Prepare environment and verify GPU - prepare: - needs: ec2 - runs-on: ${{ needs.ec2.outputs.id }} - outputs: - gpu-uuid: ${{ steps.gpu-info.outputs.uuid }} - gpu-name: ${{ steps.gpu-info.outputs.name }} - steps: - - uses: actions/checkout@v4 - - - name: Get GPU info - id: gpu-info - run: | - echo "=== Preparing GPU environment ===" - nvidia-smi - uuid=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader) - name=$(nvidia-smi --query-gpu=name --format=csv,noheader) - echo "uuid=$uuid" >> $GITHUB_OUTPUT - echo "name=$name" >> $GITHUB_OUTPUT - echo "GPU UUID: $uuid" - echo "GPU Name: $name" - - - name: Setup PyTorch environment - run: | - echo "=== Setting up PyTorch environment ===" - # The DLAMI already has PyTorch installed in a conda environment - # Set up environment for GitHub Actions to use the conda env - echo "/opt/conda/envs/pytorch/bin" >> $GITHUB_PATH - echo "CONDA_DEFAULT_ENV=pytorch" >> $GITHUB_ENV - - # Verify PyTorch is available - /opt/conda/envs/pytorch/bin/python -c "import torch; print(f'PyTorch {torch.__version__} with CUDA {torch.version.cuda}')" - echo "PyTorch environment ready" - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: | - echo "Sleeping for ${{ inputs.sleep }} seconds..." - echo "Instance IP available in GitHub Actions log for SSH" - sleep ${{ inputs.sleep }} - - # Job 2: Training simulation - train: - needs: [ec2, prepare] - runs-on: ${{ needs.ec2.outputs.id }} - steps: - - uses: actions/checkout@v4 - - - name: Verify same GPU - run: | - echo "=== Training on GPU ===" - current_uuid=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader) - if [[ "$current_uuid" == "${{ needs.prepare.outputs.gpu-uuid }}" ]]; then - echo "✅ Confirmed: Using same GPU as preparation job" - echo "GPU: ${{ needs.prepare.outputs.gpu-name }}" - else - echo "❌ ERROR: Different GPU!" - exit 1 - fi - - - name: Run training benchmark - run: | - echo "=== Running GPU Training Benchmark ===" - # Use the conda environment's Python which has PyTorch pre-installed - /opt/conda/envs/pytorch/bin/python .github/test-scripts/gpu-benchmark.py - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: sleep ${{ inputs.sleep }} - - # Job 3: Evaluation/testing - evaluate: - needs: [ec2, train] - runs-on: ${{ needs.ec2.outputs.id }} - steps: - - name: System diagnostics - run: | - echo "=== Final Evaluation ===" - echo "System Info:" - uname -a - lscpu | grep "Model name" || true - free -h - - echo -e "\nGPU Status:" - nvidia-smi --query-gpu=name,memory.total,driver_version,utilization.gpu,temperature.gpu --format=csv - - echo -e "\nDisk Usage:" - df -h / - - echo -e "\nInstance Metadata:" - curl -s -H "X-aws-ec2-metadata-token: $(curl -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 300' http://169.254.169.254/latest/api/token 2>/dev/null)" http://169.254.169.254/latest/meta-data/instance-type || echo "Metadata unavailable" - - - name: Verify multi-job reuse - run: | - echo "=== Verifying EC2 Instance Reuse ===" - echo "This job successfully ran on the same EC2 instance as the previous jobs" - echo "The instance will terminate automatically after idle timeout" - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: sleep ${{ inputs.sleep }} diff --git a/.github/workflows/demo-gpu-sweep.yml b/.github/workflows/demo-gpu-sweep.yml new file mode 100644 index 0000000..f089f08 --- /dev/null +++ b/.github/workflows/demo-gpu-sweep.yml @@ -0,0 +1,102 @@ +name: Demo – GPU instance sweep +on: + workflow_dispatch: + workflow_call: +permissions: + id-token: write # Required for AWS OIDC authentication + contents: read # Required for actions/checkout +jobs: + # Launch GPU instances with PyTorch 2.7 DLAMI + g4dn: + name: g4dn.xlarge + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: g4dn.xlarge + ec2_image_id: ami-0f20cc6143e3cdb84 # PyTorch 2.7 Ubuntu 22.04 + instance_name: "gpu-sweep/g4dn#$run_number" + secrets: inherit + g5: + name: g5.xlarge + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: g5.xlarge + ec2_image_id: ami-0f20cc6143e3cdb84 # PyTorch 2.7 Ubuntu 22.04 + instance_name: "gpu-sweep/g5#$run_number" + secrets: inherit + g6: + name: g6.xlarge + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: g6.xlarge + ec2_image_id: ami-0f20cc6143e3cdb84 # PyTorch 2.7 Ubuntu 22.04 + instance_name: "gpu-sweep/g6#$run_number" + secrets: inherit + g5g: + name: g5g.xlarge + uses: ./.github/workflows/runner.yml + with: + ec2_instance_type: g5g.xlarge + ec2_image_id: ami-030b738fa2282338b # PyTorch 2.7 Ubuntu 22.04 ARM64 + instance_name: "gpu-sweep/g5g#$run_number" + secrets: inherit + + # Test jobs for each GPU instance + test-g4dn: + name: Test g4dn.xlarge + needs: g4dn + runs-on: ${{ needs.g4dn.outputs.id }} + steps: + - name: GPU Test + run: | + nvidia-smi | grep Tesla + python3 -c "import torch; print(f'PyTorch: {torch.__version__}, CUDA: {torch.cuda.is_available()}')" + python3 -c "import torch; print(f'GPU: {torch.cuda.get_device_name(0)}')" + + test-g5: + name: Test g5.xlarge + needs: g5 + runs-on: ${{ needs.g5.outputs.id }} + steps: + - name: GPU Test + run: | + nvidia-smi | grep "NVIDIA A10" + python3 -c "import torch; print(f'PyTorch: {torch.__version__}, GPU: {torch.cuda.get_device_name(0)}')" + + test-g6: + name: Test g6.xlarge + needs: g6 + runs-on: ${{ needs.g6.outputs.id }} + steps: + - name: GPU Test + run: | + nvidia-smi + python3 -c "import torch; print(f'PyTorch: {torch.__version__}, GPU: {torch.cuda.get_device_name(0)}')" + + test-g5g: + name: Test g5g.xlarge + needs: g5g + runs-on: ${{ needs.g5g.outputs.id }} + steps: + - name: GPU Info + run: | + echo "=== GPU Instance Information ===" + echo "g5g.xlarge: AWS Graviton (ARM64) + NVIDIA T4g GPU" + nvidia-smi + echo "" + echo "=== PyTorch Version ===" + python3 -c "import torch; print(f'PyTorch: {torch.__version__}')" + python3 -c "import torch; print(f'CUDA Available: {torch.cuda.is_available()}')" + python3 -c "import torch; print(f'CUDA Version: {torch.version.cuda}')" + python3 -c "import torch; print(f'GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')" + - name: Basic GPU Test + run: | + python3 -c " + import torch + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + print(f'Using device: {device}') + if device.type == 'cuda': + x = torch.randn(1000, 1000).to(device) + y = torch.randn(1000, 1000).to(device) + z = torch.matmul(x, y) + print(f'Matrix multiplication result shape: {z.shape}') + " diff --git a/.github/workflows/demo-multi-instance.yml b/.github/workflows/demo-instances-mtx.yml similarity index 97% rename from .github/workflows/demo-multi-instance.yml rename to .github/workflows/demo-instances-mtx.yml index da0ac6a..fa8d408 100644 --- a/.github/workflows/demo-multi-instance.yml +++ b/.github/workflows/demo-instances-mtx.yml @@ -33,7 +33,7 @@ jobs: strategy: matrix: # Parse the JSON array of runner labels and use them in the matrix - runner: ${{ fromJson(needs.ec2.outputs.instances) }} + runner: ${{ fromJson(needs.ec2.outputs.runners) }} runs-on: ${{ matrix.runner }} name: Job on ${{ matrix.runner }} steps: diff --git a/.github/workflows/demo-job-seq.yml b/.github/workflows/demo-job-seq.yml deleted file mode 100644 index 0894991..0000000 --- a/.github/workflows/demo-job-seq.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Demo – multiple jobs on one EC2 instance, in sequence -# Tests that multiple jobs can run sequentially on the same EC2 instance -on: - workflow_dispatch: - inputs: - sleep: - description: "Sleep this many seconds at the end of the `prepare`, `train`, and `eval` jobs (optional, for SSH access and debugging)" - required: false - type: number - default: 0 - workflow_call: # Tested by `demos.yml` - inputs: - sleep: - required: false - type: number - default: 0 - -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. - -jobs: - ec2: - uses: ./.github/workflows/runner.yml - secrets: inherit - - prepare: - needs: ec2 - runs-on: ${{ needs.ec2.outputs.id }} - outputs: - gpu-uuid: ${{ steps.gpu-info.outputs.uuid }} - steps: - - name: Get GPU info - id: gpu-info - run: | - echo "=== Preparing GPU environment ===" - nvidia-smi - uuid=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader) - echo "uuid=$uuid" >> $GITHUB_OUTPUT - echo "GPU UUID: $uuid" - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: sleep ${{ inputs.sleep }} - - train: - needs: [ec2, prepare] - runs-on: ${{ needs.ec2.outputs.id }} - steps: - - name: Verify same GPU - run: | - echo "=== Training model on GPU ===" - current_uuid=$(nvidia-smi --query-gpu=gpu_uuid --format=csv,noheader) - if [[ "$current_uuid" == "${{ needs.prepare.outputs.gpu-uuid }}" ]]; then - echo "✅ Confirmed: Using same GPU as preparation job" - else - echo "❌ ERROR: Different GPU!" - exit 1 - fi - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: sleep ${{ inputs.sleep }} - - eval: - needs: [ec2, train] - runs-on: ${{ needs.ec2.outputs.id }} - steps: - - name: Final validation - run: | - echo "=== Evaluation complete ===" - nvidia-smi - - - name: Sleep (${{ inputs.sleep }}s) - if: ${{ inputs.sleep > 0 }} - run: sleep ${{ inputs.sleep }} diff --git a/.github/workflows/demo-multi-job.yml b/.github/workflows/demo-jobs-split.yml similarity index 95% rename from .github/workflows/demo-multi-job.yml rename to .github/workflows/demo-jobs-split.yml index 484c33a..00e7f44 100644 --- a/.github/workflows/demo-multi-job.yml +++ b/.github/workflows/demo-jobs-split.yml @@ -20,7 +20,7 @@ jobs: # First job type - runs on first instance build-job: needs: ec2 - runs-on: ${{ fromJson(needs.ec2.outputs.instances)[0] }} + runs-on: ${{ fromJson(needs.ec2.outputs.runners)[0] }} name: Build job steps: - uses: actions/checkout@v4 @@ -46,7 +46,7 @@ jobs: # Second job type - runs on second instance test-job: needs: ec2 - runs-on: ${{ fromJson(needs.ec2.outputs.instances)[1] }} + runs-on: ${{ fromJson(needs.ec2.outputs.runners)[1] }} name: Test job steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/demo-matrix-wide.yml b/.github/workflows/demo-matrix-wide.yml deleted file mode 100644 index 9681426..0000000 --- a/.github/workflows/demo-matrix-wide.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Demo – matrix on single runner -on: - workflow_dispatch: - workflow_call: # Tested by `demos.yml` -permissions: - id-token: write # Required for AWS OIDC authentication - contents: read # Required for actions/checkout; normally set by default, but must explicitly specify when defining a custom `permissions` block. -jobs: - ec2: - uses: ./.github/workflows/runner.yml - secrets: inherit - with: - ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) - test-matrix: - needs: ec2 - runs-on: ${{ needs.ec2.outputs.id }} - name: Test on ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] - steps: - - name: Test on ${{ matrix.os }} - run: | - echo "Testing on ${{ matrix.os }}" - echo "Runner: $(hostname)" - echo "Time: $(date)" - echo "Simulating work..." - sleep 10 - echo "Done!" - - name: Show system info - run: | - echo "=== System Information ===" - uname -a - echo "=== CPU Info ===" - lscpu | head -10 - echo "=== Memory Info ===" - free -h diff --git a/.github/workflows/demo-multi-runner.yml b/.github/workflows/demo-runners-mtx.yml similarity index 98% rename from .github/workflows/demo-multi-runner.yml rename to .github/workflows/demo-runners-mtx.yml index ac5618b..46d3081 100644 --- a/.github/workflows/demo-multi-runner.yml +++ b/.github/workflows/demo-runners-mtx.yml @@ -45,7 +45,7 @@ jobs: strategy: matrix: # Parse the JSON array of runner labels and use them in the matrix - runner: ${{ fromJson(needs.ec2.outputs.instances) }} + runner: ${{ fromJson(needs.ec2.outputs.runners) }} runs-on: ${{ matrix.runner }} name: Job on ${{ matrix.runner }} steps: diff --git a/.github/workflows/demos.yml b/.github/workflows/demos.yml index 60498af..4c2703c 100644 --- a/.github/workflows/demos.yml +++ b/.github/workflows/demos.yml @@ -5,24 +5,29 @@ permissions: id-token: write contents: read jobs: + # Minimal demos for quick testing + demo-dbg-minimal: + uses: ./.github/workflows/demo-dbg-minimal.yml + secrets: inherit demo-gpu-minimal: uses: ./.github/workflows/demo-gpu-minimal.yml secrets: inherit - demo-gpu-job-seq: - uses: ./.github/workflows/demo-gpu-job-seq.yml - secrets: inherit - demo-archs: - uses: ./.github/workflows/demo-archs.yml - secrets: inherit - demo-matrix-wide: - uses: ./.github/workflows/demo-matrix-wide.yml + + # Sweep demos for comprehensive testing + demo-cpu-sweep: + uses: ./.github/workflows/demo-cpu-sweep.yml secrets: inherit - demo-multi-instance: - uses: ./.github/workflows/demo-multi-instance.yml + demo-gpu-sweep: + uses: ./.github/workflows/demo-gpu-sweep.yml secrets: inherit - demo-multi-job: - uses: ./.github/workflows/demo-multi-job.yml + + # Multi-instance/runner demos + demo-instances-mtx: + uses: ./.github/workflows/demo-instances-mtx.yml secrets: inherit - demo-multi-runner: - uses: ./.github/workflows/demo-multi-runner.yml + demo-runners-mtx: + uses: ./.github/workflows/demo-runners-mtx.yml secrets: inherit + demo-jobs-split: + uses: ./.github/workflows/demo-jobs-split.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index 35dcd3b..c5eacd0 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -127,9 +127,9 @@ on: id: description: "Instance ID for runs-on (single instance)" value: ${{ jobs.launch.outputs.id }} - instances: - description: "JSON array of runner labels (for multiple instances)" - value: ${{ jobs.launch.outputs.instances }} + runners: + description: "JSON array of runner labels" + value: ${{ jobs.launch.outputs.runners }} mapping: description: "JSON object mapping instance IDs to runner labels" value: ${{ jobs.launch.outputs.mapping }} @@ -143,7 +143,7 @@ jobs: runs-on: ubuntu-latest outputs: id: ${{ steps.aws-start.outputs.label }} - instances: ${{ steps.aws-start.outputs.instances }} + runners: ${{ steps.aws-start.outputs.runners }} mapping: ${{ steps.aws-start.outputs.mapping }} steps: - name: Check EC2_LAUNCH_ROLE configuration diff --git a/action.yml b/action.yml index 4d26906..f9a1873 100644 --- a/action.yml +++ b/action.yml @@ -82,7 +82,7 @@ inputs: outputs: mapping: description: "A JSON object mapping instance IDs to unique GitHub runner labels. This is used in conjunction with the `instance_mapping` input when stopping." - instances: + runners: description: "A JSON list of the GitHub runner labels to be used in the 'runs-on' field" label: description: "The single runner label (for single instance use)" diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 00b695c..dfa2738 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -508,7 +508,7 @@ def set_instance_mapping(self, mapping: dict[str, str]): github_labels.append(labels) output("mapping", json.dumps(mapping)) - output("instances", json.dumps(github_labels)) + output("runners", json.dumps(github_labels)) # For single instance use, output simplified values if len(mapping) == 1 and self.runners_per_instance == 1: From b99370eba58ed75aba78ea3c1b74afc778807bb9 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 19:46:40 -0400 Subject: [PATCH 37/69] cpu-sweep names --- .github/workflows/demo-cpu-sweep.yml | 26 +++++++++++++------------- .github/workflows/demo-dbg-minimal.yml | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml index 7637bcd..2731f55 100644 --- a/.github/workflows/demo-cpu-sweep.yml +++ b/.github/workflows/demo-cpu-sweep.yml @@ -18,23 +18,23 @@ permissions: contents: read # Required for actions/checkout jobs: os-arch-matrix: - name: ${{ matrix.os }} ${{ matrix.version }} ${{ matrix.arch }} + name: ${{ matrix.os }} ${{ matrix.arch }} strategy: fail-fast: false matrix: include: - - { "os": Ubuntu, "version": "22.04", "arch": x86, "ami": ami-021589336d307b577, "instance_type": t3.medium } - - { "os": Ubuntu, "version": "22.04", "arch": ARM, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } - - { "os": Ubuntu, "version": "24.04", "arch": x86, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } - - { "os": Ubuntu, "version": "24.04", "arch": ARM, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } - - { "os": Debian, "version": "11", "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } - - { "os": Debian, "version": "11", "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } - - { "os": Debian, "version": "12", "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } - - { "os": Debian, "version": "12", "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } - - { "os": AL2, "version": "2", "arch": x86, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } - - { "os": AL2, "version": "2", "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } - - { "os": AL2023, "version": "2023", "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } - - { "os": AL2023, "version": "2023", "arch": ARM, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } + - { "os": Ubuntu "22.04", "arch": x86, "ami": ami-021589336d307b577, "instance_type": t3.medium } + - { "os": Ubuntu "22.04", "arch": ARM, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } + - { "os": Ubuntu "24.04", "arch": x86, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } + - { "os": Ubuntu "24.04", "arch": ARM, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } + - { "os": Debian "11", "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } + - { "os": Debian "11", "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } + - { "os": Debian "12", "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } + - { "os": Debian "12", "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } + - { "os": AL2 , "arch": x86, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } + - { "os": AL2 , "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } + - { "os": AL2023 , "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } + - { "os": AL2023 , "arch": ARM, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } uses: ./.github/workflows/demo-dbg-minimal.yml with: type: ${{ matrix.instance_type }} diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml index f48818a..7571ffc 100644 --- a/.github/workflows/demo-dbg-minimal.yml +++ b/.github/workflows/demo-dbg-minimal.yml @@ -74,7 +74,7 @@ jobs: name: Launch ${{ inputs.type }} uses: ./.github/workflows/runner.yml with: - name: '🚀' + name: 🚀 ec2_instance_type: ${{ inputs.type }} ec2_image_id: ${{ inputs.ami }} debug: ${{ inputs.debug }} @@ -88,7 +88,7 @@ jobs: test: needs: launch - name: Test Job + name: 🧪 runs-on: ${{ needs.launch.outputs.id }} steps: - name: Instance Info From c03a8c68cae5b0fca0540777264f62831ddb00cd Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 19:47:06 -0400 Subject: [PATCH 38/69] gpu-sweep conda fix --- .github/workflows/demo-gpu-sweep.yml | 30 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/demo-gpu-sweep.yml b/.github/workflows/demo-gpu-sweep.yml index f089f08..b62afaa 100644 --- a/.github/workflows/demo-gpu-sweep.yml +++ b/.github/workflows/demo-gpu-sweep.yml @@ -6,13 +6,13 @@ permissions: id-token: write # Required for AWS OIDC authentication contents: read # Required for actions/checkout jobs: - # Launch GPU instances with PyTorch 2.7 DLAMI + # Launch GPU instances with PyTorch DLAMIs g4dn: name: g4dn.xlarge uses: ./.github/workflows/runner.yml with: ec2_instance_type: g4dn.xlarge - ec2_image_id: ami-0f20cc6143e3cdb84 # PyTorch 2.7 Ubuntu 22.04 + ec2_image_id: ami-00dddcf8fefea182f # Deep Learning OSS PyTorch 2.5.1 Ubuntu 22.04 instance_name: "gpu-sweep/g4dn#$run_number" secrets: inherit g5: @@ -20,7 +20,7 @@ jobs: uses: ./.github/workflows/runner.yml with: ec2_instance_type: g5.xlarge - ec2_image_id: ami-0f20cc6143e3cdb84 # PyTorch 2.7 Ubuntu 22.04 + ec2_image_id: ami-00dddcf8fefea182f # Deep Learning OSS PyTorch 2.5.1 Ubuntu 22.04 instance_name: "gpu-sweep/g5#$run_number" secrets: inherit g6: @@ -28,7 +28,7 @@ jobs: uses: ./.github/workflows/runner.yml with: ec2_instance_type: g6.xlarge - ec2_image_id: ami-0f20cc6143e3cdb84 # PyTorch 2.7 Ubuntu 22.04 + ec2_image_id: ami-00dddcf8fefea182f # Deep Learning OSS PyTorch 2.5.1 Ubuntu 22.04 instance_name: "gpu-sweep/g6#$run_number" secrets: inherit g5g: @@ -36,7 +36,7 @@ jobs: uses: ./.github/workflows/runner.yml with: ec2_instance_type: g5g.xlarge - ec2_image_id: ami-030b738fa2282338b # PyTorch 2.7 Ubuntu 22.04 ARM64 + ec2_image_id: ami-00cbe74a3dff23b9f # Deep Learning ARM64 OSS PyTorch 2.5.1 Ubuntu 22.04 instance_name: "gpu-sweep/g5g#$run_number" secrets: inherit @@ -48,7 +48,10 @@ jobs: steps: - name: GPU Test run: | - nvidia-smi | grep Tesla + nvidia-smi + # Activate PyTorch conda environment + source /opt/conda/etc/profile.d/conda.sh + conda activate pytorch python3 -c "import torch; print(f'PyTorch: {torch.__version__}, CUDA: {torch.cuda.is_available()}')" python3 -c "import torch; print(f'GPU: {torch.cuda.get_device_name(0)}')" @@ -59,7 +62,10 @@ jobs: steps: - name: GPU Test run: | - nvidia-smi | grep "NVIDIA A10" + nvidia-smi + # Activate PyTorch conda environment + source /opt/conda/etc/profile.d/conda.sh + conda activate pytorch python3 -c "import torch; print(f'PyTorch: {torch.__version__}, GPU: {torch.cuda.get_device_name(0)}')" test-g6: @@ -70,6 +76,9 @@ jobs: - name: GPU Test run: | nvidia-smi + # Activate PyTorch conda environment + source /opt/conda/etc/profile.d/conda.sh + conda activate pytorch python3 -c "import torch; print(f'PyTorch: {torch.__version__}, GPU: {torch.cuda.get_device_name(0)}')" test-g5g: @@ -83,13 +92,18 @@ jobs: echo "g5g.xlarge: AWS Graviton (ARM64) + NVIDIA T4g GPU" nvidia-smi echo "" - echo "=== PyTorch Version ===" + echo "=== PyTorch Test ===" + # Activate PyTorch conda environment + source /opt/conda/etc/profile.d/conda.sh + conda activate pytorch python3 -c "import torch; print(f'PyTorch: {torch.__version__}')" python3 -c "import torch; print(f'CUDA Available: {torch.cuda.is_available()}')" python3 -c "import torch; print(f'CUDA Version: {torch.version.cuda}')" python3 -c "import torch; print(f'GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')" - name: Basic GPU Test run: | + source /opt/conda/etc/profile.d/conda.sh + conda activate pytorch python3 -c " import torch device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') From dbc6f2521787463f0c3a99c20c4898a171bd978c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 21:32:08 -0400 Subject: [PATCH 39/69] sweeps --- .github/workflows/demo-cpu-sweep.yml | 24 ++++++++++++------------ .github/workflows/demo-dbg-minimal.yml | 2 +- .github/workflows/demo-gpu-sweep.yml | 16 ++++++++-------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml index 2731f55..1de0b4a 100644 --- a/.github/workflows/demo-cpu-sweep.yml +++ b/.github/workflows/demo-cpu-sweep.yml @@ -23,18 +23,18 @@ jobs: fail-fast: false matrix: include: - - { "os": Ubuntu "22.04", "arch": x86, "ami": ami-021589336d307b577, "instance_type": t3.medium } - - { "os": Ubuntu "22.04", "arch": ARM, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } - - { "os": Ubuntu "24.04", "arch": x86, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } - - { "os": Ubuntu "24.04", "arch": ARM, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } - - { "os": Debian "11", "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } - - { "os": Debian "11", "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } - - { "os": Debian "12", "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } - - { "os": Debian "12", "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } - - { "os": AL2 , "arch": x86, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } - - { "os": AL2 , "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } - - { "os": AL2023 , "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } - - { "os": AL2023 , "arch": ARM, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } + - { "os": Ubuntu 22.04, "arch": x86, "ami": ami-021589336d307b577, "instance_type": t3.medium } + - { "os": Ubuntu 22.04, "arch": ARM, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } + - { "os": Ubuntu 24.04, "arch": x86, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } + - { "os": Ubuntu 24.04, "arch": ARM, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } + - { "os": Debian 11, "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } + - { "os": Debian 11, "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } + - { "os": Debian 12, "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } + - { "os": Debian 12, "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } + - { "os": AL2 , "arch": x86, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } + - { "os": AL2 , "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } + - { "os": AL2023 , "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } + - { "os": AL2023 , "arch": ARM, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } uses: ./.github/workflows/demo-dbg-minimal.yml with: type: ${{ matrix.instance_type }} diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml index 7571ffc..72c0e68 100644 --- a/.github/workflows/demo-dbg-minimal.yml +++ b/.github/workflows/demo-dbg-minimal.yml @@ -88,7 +88,7 @@ jobs: test: needs: launch - name: 🧪 + name: 🔬 runs-on: ${{ needs.launch.outputs.id }} steps: - name: Instance Info diff --git a/.github/workflows/demo-gpu-sweep.yml b/.github/workflows/demo-gpu-sweep.yml index b62afaa..a3d63af 100644 --- a/.github/workflows/demo-gpu-sweep.yml +++ b/.github/workflows/demo-gpu-sweep.yml @@ -8,7 +8,7 @@ permissions: jobs: # Launch GPU instances with PyTorch DLAMIs g4dn: - name: g4dn.xlarge + name: 🚀 g4dn.xlarge uses: ./.github/workflows/runner.yml with: ec2_instance_type: g4dn.xlarge @@ -16,7 +16,7 @@ jobs: instance_name: "gpu-sweep/g4dn#$run_number" secrets: inherit g5: - name: g5.xlarge + name: 🚀 g5.xlarge uses: ./.github/workflows/runner.yml with: ec2_instance_type: g5.xlarge @@ -24,7 +24,7 @@ jobs: instance_name: "gpu-sweep/g5#$run_number" secrets: inherit g6: - name: g6.xlarge + name: 🚀 g6.xlarge uses: ./.github/workflows/runner.yml with: ec2_instance_type: g6.xlarge @@ -32,7 +32,7 @@ jobs: instance_name: "gpu-sweep/g6#$run_number" secrets: inherit g5g: - name: g5g.xlarge + name: 🚀 g5g.xlarge uses: ./.github/workflows/runner.yml with: ec2_instance_type: g5g.xlarge @@ -42,7 +42,7 @@ jobs: # Test jobs for each GPU instance test-g4dn: - name: Test g4dn.xlarge + name: 🔬 g4dn.xlarge needs: g4dn runs-on: ${{ needs.g4dn.outputs.id }} steps: @@ -56,7 +56,7 @@ jobs: python3 -c "import torch; print(f'GPU: {torch.cuda.get_device_name(0)}')" test-g5: - name: Test g5.xlarge + name: 🔬 g5.xlarge needs: g5 runs-on: ${{ needs.g5.outputs.id }} steps: @@ -69,7 +69,7 @@ jobs: python3 -c "import torch; print(f'PyTorch: {torch.__version__}, GPU: {torch.cuda.get_device_name(0)}')" test-g6: - name: Test g6.xlarge + name: 🔬 g6.xlarge needs: g6 runs-on: ${{ needs.g6.outputs.id }} steps: @@ -82,7 +82,7 @@ jobs: python3 -c "import torch; print(f'PyTorch: {torch.__version__}, GPU: {torch.cuda.get_device_name(0)}')" test-g5g: - name: Test g5g.xlarge + name: 🔬 g5g.xlarge needs: g5g runs-on: ${{ needs.g5g.outputs.id }} steps: From de4a8a5752592ca5c69f2bfd215b32bd7f73cc64 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 22:09:15 -0400 Subject: [PATCH 40/69] `instance_name`s, `$run` --- .github/workflows/README.md | 4 ++-- .github/workflows/demo-cpu-sweep.yml | 5 +++-- .github/workflows/demo-dbg-minimal.yml | 10 +++++++++- .github/workflows/demo-gpu-sweep.yml | 8 ++++---- .github/workflows/demo-instances-mtx.yml | 2 +- .github/workflows/demo-jobs-split.yml | 2 +- .github/workflows/demo-runners-mtx.yml | 2 +- src/ec2_gha/start.py | 6 ++++-- 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3e0024a..559a447 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -54,7 +54,7 @@ Useful regression test, demonstrates and verifies features. - Creates configurable number of instances (default: 3) - Uses matrix strategy to run jobs in parallel - Each job runs on its own EC2 instance -- **Customizes `instance_name`**: `"$repo/$name-$idx (#$run_number)"` +- **Customizes `instance_name`**: `"$repo/$name#$run $idx"` - Uses `$idx` template variable (0-based index) to distinguish instances - Results in names like `"ec2-gha/multi-instance-0 (#123)"`, `"ec2-gha/multi-instance-1 (#123)"`, etc. - **Instance type:** `t3.medium` @@ -65,7 +65,7 @@ Useful regression test, demonstrates and verifies features. - Run build job on first instance - Run test job on second instance - Aggregate results from both instances -- **Customizes `instance_name`**: `"$repo/$name-$idx (#$run_number)"` +- **Customizes `instance_name`**: `"$repo/$name#$run $idx"` - Results in names like `"ec2-gha/multi-job-0 (#123)"` and `"ec2-gha/multi-job-1 (#123)"` - **Instance type:** `t3.medium` - **Use case:** Pipeline with dedicated instances per stage diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml index 1de0b4a..791965a 100644 --- a/.github/workflows/demo-cpu-sweep.yml +++ b/.github/workflows/demo-cpu-sweep.yml @@ -18,7 +18,7 @@ permissions: contents: read # Required for actions/checkout jobs: os-arch-matrix: - name: ${{ matrix.os }} ${{ matrix.arch }} + uses: ./.github/workflows/demo-dbg-minimal.yml strategy: fail-fast: false matrix: @@ -35,9 +35,10 @@ jobs: - { "os": AL2 , "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } - { "os": AL2023 , "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } - { "os": AL2023 , "arch": ARM, "ami": ami-0aa7db6294d00216f, "instance_type": t4g.small } - uses: ./.github/workflows/demo-dbg-minimal.yml + name: ${{ matrix.os }} ${{ matrix.arch }} with: type: ${{ matrix.instance_type }} ami: ${{ matrix.ami }} sleep: ${{ inputs.sleep }} + instance_name: "cpu-sweep#$run ${{ matrix.os }}-${{ matrix.arch }}" secrets: inherit diff --git a/.github/workflows/demo-dbg-minimal.yml b/.github/workflows/demo-dbg-minimal.yml index 72c0e68..0484008 100644 --- a/.github/workflows/demo-dbg-minimal.yml +++ b/.github/workflows/demo-dbg-minimal.yml @@ -42,6 +42,10 @@ on: required: false type: boolean default: true + instance_name: + description: "Instance name" + required: false + type: string workflow_call: # Called by demo-cpu-sweep, more for the "minimal", less for the "dbg" inputs: type: @@ -64,6 +68,10 @@ on: required: false type: boolean default: false + instance_name: + description: "Instance name" + required: false + type: string permissions: id-token: write # Required for AWS OIDC authentication @@ -78,7 +86,7 @@ jobs: ec2_instance_type: ${{ inputs.type }} ec2_image_id: ${{ inputs.ami }} debug: ${{ inputs.debug }} - instance_name: "debug/$name#$run_number" + instance_name: ${{ inputs.instance_name || 'debug/$name#$run' }} # `workflow_dispatch has higher defaults for these; revert to original defaults for `workflow_call` runner_registration_timeout: ${{ inputs.registration_timeout || '300' }} runner_grace_period: ${{ inputs.grace_period || '60' }} diff --git a/.github/workflows/demo-gpu-sweep.yml b/.github/workflows/demo-gpu-sweep.yml index a3d63af..0db7c9c 100644 --- a/.github/workflows/demo-gpu-sweep.yml +++ b/.github/workflows/demo-gpu-sweep.yml @@ -13,7 +13,7 @@ jobs: with: ec2_instance_type: g4dn.xlarge ec2_image_id: ami-00dddcf8fefea182f # Deep Learning OSS PyTorch 2.5.1 Ubuntu 22.04 - instance_name: "gpu-sweep/g4dn#$run_number" + instance_name: "gpu-sweep#$run g4dn" secrets: inherit g5: name: 🚀 g5.xlarge @@ -21,7 +21,7 @@ jobs: with: ec2_instance_type: g5.xlarge ec2_image_id: ami-00dddcf8fefea182f # Deep Learning OSS PyTorch 2.5.1 Ubuntu 22.04 - instance_name: "gpu-sweep/g5#$run_number" + instance_name: "gpu-sweep#$run g5" secrets: inherit g6: name: 🚀 g6.xlarge @@ -29,7 +29,7 @@ jobs: with: ec2_instance_type: g6.xlarge ec2_image_id: ami-00dddcf8fefea182f # Deep Learning OSS PyTorch 2.5.1 Ubuntu 22.04 - instance_name: "gpu-sweep/g6#$run_number" + instance_name: "gpu-sweep#$run g6" secrets: inherit g5g: name: 🚀 g5g.xlarge @@ -37,7 +37,7 @@ jobs: with: ec2_instance_type: g5g.xlarge ec2_image_id: ami-00cbe74a3dff23b9f # Deep Learning ARM64 OSS PyTorch 2.5.1 Ubuntu 22.04 - instance_name: "gpu-sweep/g5g#$run_number" + instance_name: "gpu-sweep#$run g5g" secrets: inherit # Test jobs for each GPU instance diff --git a/.github/workflows/demo-instances-mtx.yml b/.github/workflows/demo-instances-mtx.yml index fa8d408..1e9136e 100644 --- a/.github/workflows/demo-instances-mtx.yml +++ b/.github/workflows/demo-instances-mtx.yml @@ -24,7 +24,7 @@ jobs: uses: ./.github/workflows/runner.yml secrets: inherit with: - instance_name: "$repo/$name-$idx (#$run_number)" + instance_name: "$repo/$name#$run $idx" instance_count: ${{ inputs.instance_count }} ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) diff --git a/.github/workflows/demo-jobs-split.yml b/.github/workflows/demo-jobs-split.yml index 00e7f44..64f1885 100644 --- a/.github/workflows/demo-jobs-split.yml +++ b/.github/workflows/demo-jobs-split.yml @@ -13,7 +13,7 @@ jobs: uses: ./.github/workflows/runner.yml secrets: inherit with: - instance_name: "$repo/$name-$idx (#$run_number)" + instance_name: "$repo/$name#$run $idx" instance_count: "2" ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) diff --git a/.github/workflows/demo-runners-mtx.yml b/.github/workflows/demo-runners-mtx.yml index 46d3081..9c9e63c 100644 --- a/.github/workflows/demo-runners-mtx.yml +++ b/.github/workflows/demo-runners-mtx.yml @@ -33,7 +33,7 @@ jobs: uses: ./.github/workflows/runner.yml secrets: inherit with: - instance_name: "$repo/$name (#$run_number) – $idx" + instance_name: "$repo/$name #$run" instance_count: "1" runners_per_instance: ${{ inputs.runners_per_instance }} ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index dfa2738..1f2fd79 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -144,8 +144,10 @@ def _get_template_vars(self, idx: int = None) -> dict: template_vars["name"] = "unknown" template_vars["ref"] = "unknown" - # Get run number - template_vars["run_number"] = environ.get("GITHUB_RUN_NUMBER", "unknown") + # Get run number - set both run_number and run as alias + run_num = environ.get("GITHUB_RUN_NUMBER", "unknown") + template_vars["run_number"] = run_num + template_vars["run"] = run_num # Alias for shorter naming # Add instance index if provided (for multi-instance launches) if idx is not None: From 3fa02030d2fbdf4e2602a7755c62af4a11619a1c Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 22:28:38 -0400 Subject: [PATCH 41/69] docs, $run, default $name --- .github/workflows/README.md | 99 +++++++++++++----------- .github/workflows/demo-instances-mtx.yml | 1 - .github/workflows/demo-jobs-split.yml | 1 - .github/workflows/demo-runners-mtx.yml | 1 - .github/workflows/runner.yml | 3 +- CLAUDE.md | 2 +- README.md | 24 +++--- action.yml | 3 +- src/ec2_gha/defaults.py | 2 +- src/ec2_gha/start.py | 25 +++--- tests/test_start.py | 2 +- 11 files changed, 84 insertions(+), 79 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 559a447..7b87882 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -5,14 +5,16 @@ For documentation about the main workflow, [`runner.yml`](runner.yml), see [the - [`demos` – run all demo workflows](#demos) -- [GPU demos](#gpu) +- [Core demos](#core) + - [`dbg-minimal` – configurable debugging instance](#dbg-minimal) - [`gpu-minimal` – `nvidia-smi` "hello world"](#gpu-minimal) - - [`gpu-job-seq` – GPU train/test/eval (sequential jobs)](#gpu-job-seq) - - [Real-world example: Mamba installation testing](#mamba) -- [Architecture & Parallelization](#arch) - - [`archs` – launch x86 and ARM nodes](#archs) - - [`multi-instance` – launch multiple instances, use in matrix](#multi-instance) - - [`multi-job` – launch multiple instances, use individually](#multi-job) + - [`cpu-sweep` – OS/architecture matrix](#cpu-sweep) + - [`gpu-sweep` – GPU instance types with PyTorch](#gpu-sweep) +- [Parallelization](#parallel) + - [`instances-mtx` – multiple instances for parallel jobs](#instances-mtx) + - [`runners-mtx` – multiple runners on single instance](#runners-mtx) + - [`jobs-split` – different job types on separate instances](#jobs-split) +- [Real-world example: Mamba installation testing](#mamba) ## [`demos`](demos.yml) – run all demo workflows @@ -20,56 +22,65 @@ Useful regression test, demonstrates and verifies features. [![](../../img/demos%2325%201.png)][demos#25] -## GPU demos +## Core demos -### [`gpu-minimal`](demo-gpu-minimal.yml) – `nvidia-smi` "hello world" -- **Instance type:** `g4dn.xlarge` +### [`dbg-minimal`](demo-dbg-minimal.yml) – configurable debugging instance +- `workflow_dispatch` with customizable parameters (instance type, AMI, timeouts) +- Also callable via `workflow_call` (used by `cpu-sweep`) +- Extended debug mode for troubleshooting +- **Instance type:** `t3.large` (default), configurable +- **Use case:** Interactive debugging and testing -### [`gpu-job-seq`](demo-gpu-job-seq.yml) – GPU train/test/eval (sequential jobs) -- Runs 3 jobs sequentially on the same GPU instance (prepare, train, evaluate) -- Uses pre-installed PyTorch from Deep Learning AMI's conda environment -- Runs GPU benchmark with matrix operations and training simulation -- Verifies same GPU is used across all jobs -- Demonstrates instance reuse for multi-stage ML workflows +### [`gpu-minimal`](demo-gpu-minimal.yml) – `nvidia-smi` "hello world" - **Instance type:** `g4dn.xlarge` -- **Use case:** ML/AI workflow testing with GPU acceleration - -### Real-world example: [Mamba installation testing](https://github.com/Open-Athena/mamba/blob/gha/.github/workflows/install.yaml) -- Tests different versions of `mamba_ssm` package on GPU instances -- **Customizes `instance_name`**: `"$repo/$name==${{ inputs.mamba_version }} (#$run_number)"` - - Results in descriptive names like `"Open-Athena/mamba/install==2.2.5 (#123)"` - - Makes it easy to identify which version is being tested on each instance -- Uses pre-installed PyTorch from DLAMI conda environment -- **Use case:** Package compatibility testing across versions -[![](../../img/mamba%2312.png)][mamba#12] +### [`cpu-sweep`](demo-cpu-sweep.yml) – OS/architecture matrix +- Tests 12 combinations across operating systems and architectures +- **OS:** Ubuntu 22.04/24.04, Debian 11/12, AL2, AL2023 +- **Architectures:** x86 (`t3.*`) and ARM (`t4g.*`) +- Calls `dbg-minimal` for each combination +- **Use case:** Cross-platform compatibility testing -## Architecture & Parallelization +### [`gpu-sweep`](demo-gpu-sweep.yml) – GPU instance types with PyTorch +- Tests different GPU instance families +- **Instance types:** `g4dn.xlarge`, `g5.xlarge`, `g6.xlarge`, `g5g.xlarge` (ARM64 + GPU) +- Uses Deep Learning OSS PyTorch 2.5.1 AMIs +- Activates conda environment and runs PyTorch CUDA tests +- **Use case:** GPU compatibility and performance testing -### [`archs`](demo-archs.yml) – launch x86 and ARM nodes -- Verify architecture-specific behavior -- **Instance types:** `t3.medium` (x86), `t4g.medium` (ARM) +## Parallelization -### [`multi-instance`](demo-multi-instance.yml) – launch multiple instances, use in matrix +### [`instances-mtx`](demo-instances-mtx.yml) – multiple instances for parallel jobs - Creates configurable number of instances (default: 3) - Uses matrix strategy to run jobs in parallel - Each job runs on its own EC2 instance -- **Customizes `instance_name`**: `"$repo/$name#$run $idx"` - - Uses `$idx` template variable (0-based index) to distinguish instances - - Results in names like `"ec2-gha/multi-instance-0 (#123)"`, `"ec2-gha/multi-instance-1 (#123)"`, etc. - **Instance type:** `t3.medium` -- **Use case:** Parallel test execution - -### [`multi-job`](demo-multi-job.yml) – launch multiple instances, use individually -- Launch 2 instances -- Run build job on first instance -- Run test job on second instance -- Aggregate results from both instances -- **Customizes `instance_name`**: `"$repo/$name#$run $idx"` - - Results in names like `"ec2-gha/multi-job-0 (#123)"` and `"ec2-gha/multi-job-1 (#123)"` +- **Use case:** Parallel test execution, distributed builds + +### [`runners-mtx`](demo-runners-mtx.yml) – multiple runners on single instance +- Configurable runners per instance (default: 3) +- All runners share the same instance resources +- Demonstrates resource-efficient parallel execution +- **Instance type:** `t3.xlarge` (larger instance for multiple runners) +- **Use case:** Shared environment testing, resource optimization + +### [`jobs-split`](demo-jobs-split.yml) – different job types on separate instances +- Launches 2 instances +- Build job runs on first instance (`runners[0]`) +- Test job runs on second instance (`runners[1]`) +- Demonstrates targeted job placement - **Instance type:** `t3.medium` - **Use case:** Pipeline with dedicated instances per stage +## Real-world example: [Mamba installation testing](https://github.com/Open-Athena/mamba/blob/gha/.github/workflows/install.yaml) +- Tests different versions of `mamba_ssm` package on GPU instances +- **Customizes `instance_name`**: `"$repo/$name==${{ inputs.mamba_version }} (#$run)"` + - Results in descriptive names like `"mamba/install==2.2.5 (#123)"` + - Makes it easy to identify which version is being tested on each instance +- Uses pre-installed PyTorch from DLAMI conda environment +- **Use case:** Package compatibility testing across versions + +[![](../../img/mamba%2312.png)][mamba#12] + [mamba#12]: https://github.com/Open-Athena/mamba/actions/runs/16972369660/ [demos#25]: https://github.com/Open-Athena/ec2-gha/actions/runs/17004697889 - diff --git a/.github/workflows/demo-instances-mtx.yml b/.github/workflows/demo-instances-mtx.yml index 1e9136e..9571353 100644 --- a/.github/workflows/demo-instances-mtx.yml +++ b/.github/workflows/demo-instances-mtx.yml @@ -24,7 +24,6 @@ jobs: uses: ./.github/workflows/runner.yml secrets: inherit with: - instance_name: "$repo/$name#$run $idx" instance_count: ${{ inputs.instance_count }} ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) diff --git a/.github/workflows/demo-jobs-split.yml b/.github/workflows/demo-jobs-split.yml index 64f1885..b930753 100644 --- a/.github/workflows/demo-jobs-split.yml +++ b/.github/workflows/demo-jobs-split.yml @@ -13,7 +13,6 @@ jobs: uses: ./.github/workflows/runner.yml secrets: inherit with: - instance_name: "$repo/$name#$run $idx" instance_count: "2" ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) diff --git a/.github/workflows/demo-runners-mtx.yml b/.github/workflows/demo-runners-mtx.yml index 9c9e63c..be6fa79 100644 --- a/.github/workflows/demo-runners-mtx.yml +++ b/.github/workflows/demo-runners-mtx.yml @@ -33,7 +33,6 @@ jobs: uses: ./.github/workflows/runner.yml secrets: inherit with: - instance_name: "$repo/$name #$run" instance_count: "1" runners_per_instance: ${{ inputs.runners_per_instance }} ec2_image_id: ami-0e86e20dae9224db8 # Ubuntu 24.04 LTS x86_64 (us-east-1) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index c5eacd0..5c40a89 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -86,10 +86,9 @@ on: type: string default: "1" instance_name: - description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number" + description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run (number), $idx (0-based instance index for multi-instance launches). Default: $repo/$name#$run (or $repo/$name#$run $idx for multi-instance)" required: false type: string - default: "$repo/$name#$run_number" max_instance_lifetime: description: "Maximum instance lifetime in minutes before automatic shutdown (falls back to vars.MAX_INSTANCE_LIFETIME, then 360 = 6 hours)" required: false diff --git a/CLAUDE.md b/CLAUDE.md index d191249..f3b9678 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,7 +121,7 @@ The runner uses a polling-based approach to determine when to terminate: ### AWS Resource Tagging By default, launched EC2 instances are Tagged with: -- `Name`: `f"{repo}/{workflow}#{run_number}"` +- `Name`: `f"{repo}/{workflow}#{run}"` - `Repository`: GitHub repository name - `Workflow`: Workflow name - `URL`: Direct link to the GitHub Actions run diff --git a/README.md b/README.md index 2ff85f1..c924d99 100644 --- a/README.md +++ b/README.md @@ -62,16 +62,13 @@ Example workflows demonstrating ec2-gha capabilities are in [`.github/workflows/ [![](img/demos%2325%201.png)][demos#25] -### GPU Workflows -- [`demo-gpu-minimal.yml`](.github/workflows/demo-gpu-minimal.yml) - Minimal GPU test with `nvidia-smi` -- [`demo-gpu-job-seq.yml`](.github/workflows/demo-gpu-job-seq.yml) - Sequential ML workflow (prepare→train→evaluate) using pre-installed PyTorch from DLAMI -- Real-world example: [Mamba installation testing](https://github.com/Open-Athena/mamba/blob/gha/.github/workflows/install.yaml) - Shows `instance_name` customization for version-specific testing - -### Architecture & Parallelization -- [`demo-archs.yml`](.github/workflows/demo-archs.yml) - Cross-architecture testing (x86 and ARM) -- [`demo-multi-instance.yml`](.github/workflows/demo-multi-instance.yml) - Parallel jobs on multiple instances -- [`demo-multi-job.yml`](.github/workflows/demo-multi-job.yml) - Different job types on separate instances -- [`demo-matrix-wide.yml`](.github/workflows/demo-matrix-wide.yml) - Wide matrix strategy across many instances +- [`demo-dbg-minimal.yml`](.github/workflows/demo-dbg-minimal.yml) - Configurable debugging instance +- [`demo-gpu-minimal.yml`](.github/workflows/demo-gpu-minimal.yml) - Basic GPU test +- [`demo-cpu-sweep.yml`](.github/workflows/demo-cpu-sweep.yml) - OS/arch matrix (Ubuntu, Debian, AL2/AL2023 on x86/ARM) +- [`demo-gpu-sweep.yml`](.github/workflows/demo-gpu-sweep.yml) - GPU instances (g4dn, g5, g6, g5g) with PyTorch +- [`demo-instances-mtx.yml`](.github/workflows/demo-instances-mtx.yml) - Multiple instances for parallel jobs +- [`demo-runners-mtx.yml`](.github/workflows/demo-runners-mtx.yml) - Multiple runners on single instance +- [`demo-jobs-split.yml`](.github/workflows/demo-jobs-split.yml) - Different job types on separate instances ### Test Suite - [`demos.yml`](.github/workflows/demos.yml) - Runs all demos for regression testing @@ -119,7 +116,7 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): - `ec2_instance_type` - Instance type (default: `t3.medium`) - `ec2_key_name` - EC2 key pair name (for [SSH access]) - `instance_count` - Number of instances to create (default: 1, for parallel jobs) -- `instance_name` - Name tag template for EC2 instances. Uses Python string.Template format with variables: `$repo`, `$name` (workflow filename stem), `$workflow` (full workflow name), `$ref`, `$run_number`, `$idx` (0-based instance index for multi-instance launches). Default: `$repo/$name#$run_number` +- `instance_name` - Name tag template for EC2 instances. Uses Python string.Template format with variables: `$repo`, `$name` (workflow filename stem), `$workflow` (full workflow name), `$ref`, `$run` (number), `$idx` (0-based instance index for multi-instance launches). Default: `$repo/$name#$run` (or `$repo/$name#$run $idx` for multi-instance) - `ec2_root_device_size` - Root device size in GB (default: 0 = use AMI default) - `ec2_security_group_id` - Security group ID (required for [SSH access], should expose inbound port 22) - `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing) @@ -204,7 +201,7 @@ jobs: steps: - run: echo "Evaluating results" ``` -(see also [demo-job-seq], [demo-archs], [demo-matrix-wide]) +(see also demo workflows in [`.github/workflows/`](.github/workflows/)) ### Termination logic @@ -677,9 +674,6 @@ Here's a diff porting [ec2-github-runner][machulav/ec2-github-runner]'s README [ ``` [`runner.yml`]: .github/workflows/runner.yml -[demo-job-seq]: .github/workflows/demo-job-seq.yml -[demo-archs]: .github/workflows/demo-archs.yml -[demo-matrix-wide]: .github/workflows/demo-matrix-wide.yml [aws-actions/configure-aws-credentials]: https://github.com/aws-actions/configure-aws-credentials [hooks]: https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/run-scripts [omsf/start-aws-gha-runner]: https://github.com/omsf/start-aws-gha-runner diff --git a/action.yml b/action.yml index f9a1873..2befcde 100644 --- a/action.yml +++ b/action.yml @@ -51,9 +51,8 @@ inputs: required: false default: "1" instance_name: - description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run_number, $idx (0-based instance index for multi-instance launches)" + description: "Name tag template for EC2 instances. Uses Python string.Template format with variables: $repo, $name (workflow filename stem), $workflow (full workflow name), $ref, $run (number), $idx (0-based instance index for multi-instance launches). Default: $repo/$name#$run (or $repo/$name#$run $idx for multi-instance)" required: false - default: "$repo/$name#$run_number" max_instance_lifetime: description: "Maximum instance lifetime in minutes before automatic shutdown (default 360 = 6 hours)" required: false diff --git a/src/ec2_gha/defaults.py b/src/ec2_gha/defaults.py index 40e9cb5..55233ee 100644 --- a/src/ec2_gha/defaults.py +++ b/src/ec2_gha/defaults.py @@ -11,7 +11,7 @@ EC2_INSTANCE_TYPE = "t3.medium" # Instance naming default template -INSTANCE_NAME = "$repo/$name#$run_number" +INSTANCE_NAME = "$repo/$name#$run" # Default instance count INSTANCE_COUNT = 1 diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 1f2fd79..2fb91b7 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -144,10 +144,10 @@ def _get_template_vars(self, idx: int = None) -> dict: template_vars["name"] = "unknown" template_vars["ref"] = "unknown" - # Get run number - set both run_number and run as alias + # `$run` (number) run_num = environ.get("GITHUB_RUN_NUMBER", "unknown") - template_vars["run_number"] = run_num - template_vars["run"] = run_num # Alias for shorter naming + template_vars["run"] = run_num + template_vars["run_number"] = run_num # Legacy alias # Add instance index if provided (for multi-instance launches) if idx is not None: @@ -357,6 +357,12 @@ def create_instances(self) -> dict[str, str]: # Determine which tokens to use tokens_to_use = self.grouped_runner_tokens if self.grouped_runner_tokens else [[t] for t in self.gh_runner_tokens] + # Determine default instance_name based on instance count + instance_count = len(tokens_to_use) + default_instance_name = "$repo/$name#$run" + if instance_count > 1: + default_instance_name = "$repo/$name#$run $idx" + for idx, instance_tokens in enumerate(tokens_to_use): # Generate labels and tokens for all runners on this instance runner_configs = [] @@ -376,13 +382,12 @@ def create_instances(self) -> dict[str, str]: runner_labels = "|".join(config["labels"] for config in runner_configs) # Generate instance name using template variables - if self.instance_name: - from string import Template - template_vars = self._get_template_vars(idx) - name_template = Template(self.instance_name) - instance_name_value = name_template.safe_substitute(**template_vars) - else: - instance_name_value = "" + from string import Template + template_vars = self._get_template_vars(idx) + # Use provided instance_name or the smart default + name_pattern = self.instance_name if self.instance_name else default_instance_name + name_template = Template(name_pattern) + instance_name_value = name_template.safe_substitute(**template_vars) user_data_params = { "cloudwatch_logs_group": self.cloudwatch_logs_group, diff --git a/tests/test_start.py b/tests/test_start.py index 76f6548..68d385b 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -128,7 +128,7 @@ def test_build_aws_params_with_idx(complete_params, aws_params_user_data, github params_without_tags = complete_params.copy() params_without_tags['tags'] = [] # Add instance_name template for testing - params_without_tags['instance_name'] = '$repo/$name-$idx#$run_number' + params_without_tags['instance_name'] = '$repo/$name#$run $idx' aws = StartAWS(**params_without_tags) params = aws._build_aws_params(user_data_params, idx=0) From 1fb4726788f5f3206386630ee4525045975896d4 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 22:55:07 -0400 Subject: [PATCH 42/69] `outputs.matrix`, rm `runners,mapping` --- .github/workflows/demo-instances-mtx.yml | 11 ++++---- .github/workflows/demo-jobs-split.yml | 4 +-- .github/workflows/demo-runners-mtx.yml | 12 +++++---- .github/workflows/runner.yml | 12 +++------ action.yml | 6 ++--- src/ec2_gha/start.py | 32 ++++++++++++++++++------ 6 files changed, 46 insertions(+), 31 deletions(-) diff --git a/.github/workflows/demo-instances-mtx.yml b/.github/workflows/demo-instances-mtx.yml index 9571353..defebc2 100644 --- a/.github/workflows/demo-instances-mtx.yml +++ b/.github/workflows/demo-instances-mtx.yml @@ -31,16 +31,17 @@ jobs: needs: ec2 strategy: matrix: - # Parse the JSON array of runner labels and use them in the matrix - runner: ${{ fromJson(needs.ec2.outputs.runners) }} - runs-on: ${{ matrix.runner }} - name: Job on ${{ matrix.runner }} + # Parse the JSON array of runner objects for the matrix + runner: ${{ fromJson(needs.ec2.outputs.matrix) }} + runs-on: ${{ matrix.runner.id }} + name: Job #${{ matrix.runner.idx }} (instance ${{ matrix.runner.instance_idx }}) steps: - uses: actions/checkout@v4 - name: Instance info run: | - echo "Running on instance with label: ${{ matrix.runner }}" + echo "Running on instance with label: ${{ matrix.runner.id }}" + echo "Job index: ${{ matrix.runner.idx }}" echo "Hostname: $(hostname)" echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" diff --git a/.github/workflows/demo-jobs-split.yml b/.github/workflows/demo-jobs-split.yml index b930753..ba0f567 100644 --- a/.github/workflows/demo-jobs-split.yml +++ b/.github/workflows/demo-jobs-split.yml @@ -19,7 +19,7 @@ jobs: # First job type - runs on first instance build-job: needs: ec2 - runs-on: ${{ fromJson(needs.ec2.outputs.runners)[0] }} + runs-on: ${{ fromJson(needs.ec2.outputs.matrix)[0].id }} name: Build job steps: - uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: # Second job type - runs on second instance test-job: needs: ec2 - runs-on: ${{ fromJson(needs.ec2.outputs.runners)[1] }} + runs-on: ${{ fromJson(needs.ec2.outputs.matrix)[1].id }} name: Test job steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/demo-runners-mtx.yml b/.github/workflows/demo-runners-mtx.yml index be6fa79..501ee66 100644 --- a/.github/workflows/demo-runners-mtx.yml +++ b/.github/workflows/demo-runners-mtx.yml @@ -43,16 +43,18 @@ jobs: needs: ec2 strategy: matrix: - # Parse the JSON array of runner labels and use them in the matrix - runner: ${{ fromJson(needs.ec2.outputs.runners) }} - runs-on: ${{ matrix.runner }} - name: Job on ${{ matrix.runner }} + # Parse the JSON array of runner objects for the matrix + runner: ${{ fromJson(needs.ec2.outputs.matrix) }} + runs-on: ${{ matrix.runner.id }} + name: 'Parallel job ${{ matrix.runner.idx }} (${{ matrix.runner.id }})' steps: - uses: actions/checkout@v4 - name: Runner info run: | - echo "Running on runner with label: ${{ matrix.runner }}" + echo "Running on runner with label: ${{ matrix.runner.id }}" + echo "Runner index: ${{ matrix.runner.idx }}" + echo "Runner index on instance: ${{ matrix.runner.runner_idx }}" echo "Hostname: $(hostname)" echo "Instance ID: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" echo "Instance type: $(curl -s http://169.254.169.254/latest/meta-data/instance-type)" diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index 5c40a89..1eed753 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -126,12 +126,9 @@ on: id: description: "Instance ID for runs-on (single instance)" value: ${{ jobs.launch.outputs.id }} - runners: - description: "JSON array of runner labels" - value: ${{ jobs.launch.outputs.runners }} - mapping: - description: "JSON object mapping instance IDs to runner labels" - value: ${{ jobs.launch.outputs.mapping }} + matrix: + description: "JSON array of objects for matrix strategies" + value: ${{ jobs.launch.outputs.matrix }} permissions: id-token: write # Required for AWS OIDC @@ -142,8 +139,7 @@ jobs: runs-on: ubuntu-latest outputs: id: ${{ steps.aws-start.outputs.label }} - runners: ${{ steps.aws-start.outputs.runners }} - mapping: ${{ steps.aws-start.outputs.mapping }} + matrix: ${{ steps.aws-start.outputs.matrix }} steps: - name: Check EC2_LAUNCH_ROLE configuration run: | diff --git a/action.yml b/action.yml index 2befcde..3720771 100644 --- a/action.yml +++ b/action.yml @@ -79,10 +79,8 @@ inputs: description: "SSH public key to add to authorized_keys for debugging access" required: false outputs: - mapping: - description: "A JSON object mapping instance IDs to unique GitHub runner labels. This is used in conjunction with the `instance_mapping` input when stopping." - runners: - description: "A JSON list of the GitHub runner labels to be used in the 'runs-on' field" + matrix: + description: "A JSON array of objects for matrix strategies. Each object has: idx (overall 0-based index), id (runner label), instance_id, instance_idx (0-based instance index), runner_idx (0-based runner index within instance)" label: description: "The single runner label (for single instance use)" instance-id: diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 2fb91b7..22d1c09 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -506,16 +506,34 @@ def set_instance_mapping(self, mapping: dict[str, str]): A dictionary of instance IDs and labels. """ - # Flatten all labels from all instances - github_labels = [] - for labels in mapping.values(): + # Build matrix objects for all runners + matrix_objects = [] + idx = 0 + + for instance_idx, (instance_id, labels) in enumerate(mapping.items()): if isinstance(labels, list): - github_labels.extend(labels) + # Multiple runners on this instance + for runner_idx, label in enumerate(labels): + matrix_objects.append({ + "idx": idx, + "id": label, + "instance_id": instance_id, + "instance_idx": instance_idx, + "runner_idx": runner_idx + }) + idx += 1 else: - github_labels.append(labels) + # Single runner on this instance + matrix_objects.append({ + "idx": idx, + "id": labels, + "instance_id": instance_id, + "instance_idx": instance_idx, + "runner_idx": 0 + }) + idx += 1 - output("mapping", json.dumps(mapping)) - output("runners", json.dumps(github_labels)) + output("matrix", json.dumps(matrix_objects)) # For single instance use, output simplified values if len(mapping) == 1 and self.runners_per_instance == 1: From 15cb12790329c1fb1f088c841481a5e96541dd06 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 23:01:18 -0400 Subject: [PATCH 43/69] rename: `outputs.{matrix,mtx}` --- .github/workflows/README.md | 4 ++-- .github/workflows/demo-instances-mtx.yml | 2 +- .github/workflows/demo-jobs-split.yml | 4 ++-- .github/workflows/demo-runners-mtx.yml | 2 +- .github/workflows/runner.yml | 6 +++--- CLAUDE.md | 2 +- README.md | 15 +++++++-------- action.yml | 2 +- src/ec2_gha/start.py | 2 +- 9 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 7b87882..1aa8e18 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -66,8 +66,8 @@ Useful regression test, demonstrates and verifies features. ### [`jobs-split`](demo-jobs-split.yml) – different job types on separate instances - Launches 2 instances -- Build job runs on first instance (`runners[0]`) -- Test job runs on second instance (`runners[1]`) +- Build job runs on first instance +- Test job runs on second instance - Demonstrates targeted job placement - **Instance type:** `t3.medium` - **Use case:** Pipeline with dedicated instances per stage diff --git a/.github/workflows/demo-instances-mtx.yml b/.github/workflows/demo-instances-mtx.yml index defebc2..dec379c 100644 --- a/.github/workflows/demo-instances-mtx.yml +++ b/.github/workflows/demo-instances-mtx.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: # Parse the JSON array of runner objects for the matrix - runner: ${{ fromJson(needs.ec2.outputs.matrix) }} + runner: ${{ fromJson(needs.ec2.outputs.mtx) }} runs-on: ${{ matrix.runner.id }} name: Job #${{ matrix.runner.idx }} (instance ${{ matrix.runner.instance_idx }}) steps: diff --git a/.github/workflows/demo-jobs-split.yml b/.github/workflows/demo-jobs-split.yml index ba0f567..40b3782 100644 --- a/.github/workflows/demo-jobs-split.yml +++ b/.github/workflows/demo-jobs-split.yml @@ -19,7 +19,7 @@ jobs: # First job type - runs on first instance build-job: needs: ec2 - runs-on: ${{ fromJson(needs.ec2.outputs.matrix)[0].id }} + runs-on: ${{ fromJson(needs.ec2.outputs.mtx)[0].id }} name: Build job steps: - uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: # Second job type - runs on second instance test-job: needs: ec2 - runs-on: ${{ fromJson(needs.ec2.outputs.matrix)[1].id }} + runs-on: ${{ fromJson(needs.ec2.outputs.mtx)[1].id }} name: Test job steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/demo-runners-mtx.yml b/.github/workflows/demo-runners-mtx.yml index 501ee66..775ca21 100644 --- a/.github/workflows/demo-runners-mtx.yml +++ b/.github/workflows/demo-runners-mtx.yml @@ -44,7 +44,7 @@ jobs: strategy: matrix: # Parse the JSON array of runner objects for the matrix - runner: ${{ fromJson(needs.ec2.outputs.matrix) }} + runner: ${{ fromJson(needs.ec2.outputs.mtx) }} runs-on: ${{ matrix.runner.id }} name: 'Parallel job ${{ matrix.runner.idx }} (${{ matrix.runner.id }})' steps: diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index 1eed753..cfec631 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -126,9 +126,9 @@ on: id: description: "Instance ID for runs-on (single instance)" value: ${{ jobs.launch.outputs.id }} - matrix: + mtx: description: "JSON array of objects for matrix strategies" - value: ${{ jobs.launch.outputs.matrix }} + value: ${{ jobs.launch.outputs.mtx }} permissions: id-token: write # Required for AWS OIDC @@ -139,7 +139,7 @@ jobs: runs-on: ubuntu-latest outputs: id: ${{ steps.aws-start.outputs.label }} - matrix: ${{ steps.aws-start.outputs.matrix }} + mtx: ${{ steps.aws-start.outputs.mtx }} steps: - name: Check EC2_LAUNCH_ROLE configuration run: | diff --git a/CLAUDE.md b/CLAUDE.md index f3b9678..696a6db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,7 +44,7 @@ ruff format src/ - AWS/EC2 configs (instance type, AMI, optional CloudWatch log group, keypair/pubkey for SSH-debugging, etc.) - GitHub runner configurations (timeouts / poll intervals, labels, etc.) - Outputs: - - `mapping`, `instances` + - `mtx` (array of objects for matrix strategies) - When only one instance/runner is created, also outputs `label` and `instance-id` ### Core Python Modules diff --git a/README.md b/README.md index c924d99..7457e40 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,7 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): | Name | Description | |------|--------------------------------------------------------------------------| | id | Single runner label for `runs-on` (when `instance_count=1`) | -| instances | JSON array of runner labels (for use with matrix strategy) | -| mapping | JSON object mapping instance IDs to labels (for debugging) | +| mtx | JSON array of objects for matrix strategies (each has: idx, id, instance_id, instance_idx, runner_idx) | ## Technical Details @@ -160,10 +159,10 @@ jobs: needs: ec2 strategy: matrix: - runner: ${{ fromJson(needs.ec2.outputs.instances) }} - runs-on: ${{ matrix.runner }} + runner: ${{ fromJson(needs.ec2.outputs.mtx) }} + runs-on: ${{ matrix.runner.id }} steps: - - run: echo "Running on ${{ matrix.runner }}" + - run: echo "Running on runner ${{ matrix.runner.idx }} (instance ${{ matrix.runner.instance_idx }})" ``` Each instance gets a unique runner label and can execute jobs independently. This is useful for: @@ -185,19 +184,19 @@ jobs: prepare: needs: ec2 - runs-on: ${{ needs.ec2.outputs.instance }} + runs-on: ${{ needs.ec2.outputs.id }} steps: - run: echo "Preparing environment" train: needs: [ec2, prepare] - runs-on: ${{ needs.ec2.outputs.instance }} + runs-on: ${{ needs.ec2.outputs.id }} steps: - run: echo "Training model" evaluate: needs: [ec2, train] - runs-on: ${{ needs.ec2.outputs.instance }} + runs-on: ${{ needs.ec2.outputs.id }} steps: - run: echo "Evaluating results" ``` diff --git a/action.yml b/action.yml index 3720771..e6caa5b 100644 --- a/action.yml +++ b/action.yml @@ -79,7 +79,7 @@ inputs: description: "SSH public key to add to authorized_keys for debugging access" required: false outputs: - matrix: + mtx: description: "A JSON array of objects for matrix strategies. Each object has: idx (overall 0-based index), id (runner label), instance_id, instance_idx (0-based instance index), runner_idx (0-based runner index within instance)" label: description: "The single runner label (for single instance use)" diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 22d1c09..8004144 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -533,7 +533,7 @@ def set_instance_mapping(self, mapping: dict[str, str]): }) idx += 1 - output("matrix", json.dumps(matrix_objects)) + output("mtx", json.dumps(matrix_objects)) # For single instance use, output simplified values if len(mapping) == 1 and self.runners_per_instance == 1: From 34cc7839615304738edf9788d888b4bf6bf89a42 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 23:09:52 -0400 Subject: [PATCH 44/69] `Open-Athena/gha-runner@v1` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 787f117..6a00a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Start an AWS GitHub Actions Runner" readme = "README.md" requires-python = ">=3.12" authors = [{ name = "Ethan Holz", email = "ethan.holz@omsf.io" }] -dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@rw/dns"] +dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@v1"] [project.optional-dependencies] test = ["pytest", "pytest-cov", "moto[ec2]", "responses", "ruff", "syrupy"] From dd435417ae62d6943e35002b77e11ceaee270ab1 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 23:32:47 -0400 Subject: [PATCH 45/69] skip `installdependencies.sh` if deps are present --- src/ec2_gha/templates/shared-functions.sh | 44 ++++--- tests/__snapshots__/test_start.ambr | 142 ++++++++++++---------- tests/test_start.py | 8 +- 3 files changed, 102 insertions(+), 92 deletions(-) diff --git a/src/ec2_gha/templates/shared-functions.sh b/src/ec2_gha/templates/shared-functions.sh index 589b8e1..7137467 100644 --- a/src/ec2_gha/templates/shared-functions.sh +++ b/src/ec2_gha/templates/shared-functions.sh @@ -118,30 +118,28 @@ configure_runner() { cd "$runner_dir" tar -xzf /tmp/runner.tar.gz - # Install dependencies + # Install dependencies if needed if [ -f ./bin/installdependencies.sh ]; then - log "Installing dependencies..." - # Don't let installdependencies.sh failure trigger ERR trap - set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 - local deps_result=$? - set -e - - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing required packages manually..." - # Install runner dependencies based on package manager - if command -v dnf >/dev/null 2>&1; then - # Fedora/RHEL 8+/AL2023 - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - # RHEL 7/CentOS 7/AL2 - sudo yum install -y libicu >/dev/null 2>&1 || true - # Note: lttng-ust often not available in standard repos for AL2 - elif command -v apt-get >/dev/null 2>&1; then - # Debian/Ubuntu - wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + # Quick check for common AMIs with pre-installed deps + if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + log "Dependencies exist, skipping install" + else + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi fi fi fi diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index d738a64..f806a00 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -197,22 +197,25 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 - local deps_result=$? - set -e - - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing required packages manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + log "Dependencies exist, skipping install" + else + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi fi fi fi @@ -550,7 +553,7 @@ 'Tags': list([ dict({ 'Key': 'Name', - 'Value': 'ec2-gha/test-0#42', + 'Value': 'ec2-gha/test#42 0', }), dict({ 'Key': 'Repository', @@ -724,22 +727,25 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 - local deps_result=$? - set -e - - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing required packages manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + log "Dependencies exist, skipping install" + else + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi fi fi fi @@ -1215,22 +1221,25 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 - local deps_result=$? - set -e - - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing required packages manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + log "Dependencies exist, skipping install" + else + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi fi fi fi @@ -1705,22 +1714,25 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 - local deps_result=$? - set -e - - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing required packages manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + log "Dependencies exist, skipping install" + else + log "Installing dependencies..." + set +e + sudo ./bin/installdependencies.sh >/dev/null 2>&1 + local deps_result=$? + set -e + if [ $deps_result -ne 0 ]; then + log "Dependencies script failed, installing manually..." + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libicu >/dev/null 2>&1 || true + elif command -v apt-get >/dev/null 2>&1; then + wait_for_dpkg_lock + sudo apt-get update >/dev/null 2>&1 || true + sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + fi fi fi fi diff --git a/tests/test_start.py b/tests/test_start.py index 68d385b..e370de4 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -415,8 +415,8 @@ def test_set_instance_mapping(aws, monkeypatch): with patch("builtins.open", mock_file): aws.set_instance_mapping(mapping) - # Should be called 4 times for single instance (mapping, instances, instance-id, label) - assert mock_file.call_count == 4 + # Should be called 3 times for single instance (mtx, instance-id, label) + assert mock_file.call_count == 3 assert all(call[0][0] == "mock_output_file" for call in mock_file.call_args_list) @@ -428,6 +428,6 @@ def test_set_instance_mapping_multiple(aws, monkeypatch): with patch("builtins.open", mock_file): aws.set_instance_mapping(mapping) - # Should be called 2 times for multiple instances (mapping, instances only) - assert mock_file.call_count == 2 + # Should be called 1 time for multiple instances (mtx only) + assert mock_file.call_count == 1 assert all(call[0][0] == "mock_output_file" for call in mock_file.call_args_list) From 6193a32b53a9c5168227313a60a973ac117ff557 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 20 Aug 2025 23:34:16 -0400 Subject: [PATCH 46/69] cpu-sweep name tweak --- .github/workflows/demo-cpu-sweep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml index 791965a..2f4e243 100644 --- a/.github/workflows/demo-cpu-sweep.yml +++ b/.github/workflows/demo-cpu-sweep.yml @@ -40,5 +40,5 @@ jobs: type: ${{ matrix.instance_type }} ami: ${{ matrix.ami }} sleep: ${{ inputs.sleep }} - instance_name: "cpu-sweep#$run ${{ matrix.os }}-${{ matrix.arch }}" + instance_name: 'cpu-sweep#$run ${{ matrix.os }} ${{ matrix.arch }}' secrets: inherit From 83b77b4a6db19340e15326f155abedd719c1cbd2 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 7 Sep 2025 18:54:44 -0400 Subject: [PATCH 47/69] `requires-python = ">= 3.10"` --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6a00a00..7281398 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "ec2_gha" version = "1.0.0" description = "Start an AWS GitHub Actions Runner" readme = "README.md" -requires-python = ">=3.12" +requires-python = ">=3.10" authors = [{ name = "Ethan Holz", email = "ethan.holz@omsf.io" }] dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@v1"] From 404c6f913009094c6b64266debd3952dd7f2aa51 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 7 Sep 2025 18:54:55 -0400 Subject: [PATCH 48/69] add uv.lock --- uv.lock | 766 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 766 insertions(+) create mode 100644 uv.lock diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..8851867 --- /dev/null +++ b/uv.lock @@ -0,0 +1,766 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "boto3" +version = "1.40.25" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/36/de7e622fd7907faec3823eaee7299b55130f577a4ba609717a290e9f3897/boto3-1.40.25.tar.gz", hash = "sha256:debfa4b2c67492d53629a52c999d71cddc31041a8b62ca1a8b1fb60fb0712ee1", size = 111534, upload-time = "2025-09-05T19:23:21.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/9a/6b280f01f5ec7e812ac8be9803bf52868b190e15c500bee3319d9d68eb34/boto3-1.40.25-py3-none-any.whl", hash = "sha256:d39bc3deb6780d910f00580837b720132055b0604769fd978780865ed3c019ea", size = 139325, upload-time = "2025-09-05T19:23:20.551Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.25" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/ba/7faa7e1061c2d2d60700815928ec0e5a7eeb83c5311126eccc6125e1797b/botocore-1.40.25.tar.gz", hash = "sha256:41fd186018a48dc517a4312a8d3085d548cb3fb1f463972134140bf7ee55a397", size = 14331329, upload-time = "2025-09-05T19:23:12.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/e5/4c32b35109bc3f8f8ebe3d78f952d2bf702bacce975a45997cc268c11860/botocore-1.40.25-py3-none-any.whl", hash = "sha256:5603ea9955cd31974446f0b5688911a5dad71fbdfbf7457944cda8a83fcf2a9e", size = 14003384, upload-time = "2025-09-05T19:23:09.731Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191, upload-time = "2024-09-04T20:43:30.027Z" }, + { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592, upload-time = "2024-09-04T20:43:32.108Z" }, + { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024, upload-time = "2024-09-04T20:43:34.186Z" }, + { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188, upload-time = "2024-09-04T20:43:36.286Z" }, + { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571, upload-time = "2024-09-04T20:43:38.586Z" }, + { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687, upload-time = "2024-09-04T20:43:40.084Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211, upload-time = "2024-09-04T20:43:41.526Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325, upload-time = "2024-09-04T20:43:43.117Z" }, + { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784, upload-time = "2024-09-04T20:43:45.256Z" }, + { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564, upload-time = "2024-09-04T20:43:46.779Z" }, + { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804, upload-time = "2024-09-04T20:43:48.186Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299, upload-time = "2024-09-04T20:43:49.812Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, + { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, + { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, + { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, + { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, + { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, + { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, + { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, + { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, + { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, + { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, + { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, + { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, + { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, + { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, + { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, + { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, + { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, + { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, + { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, + { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, + { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, + { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.10.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/70/025b179c993f019105b79575ac6edb5e084fb0f0e63f15cdebef4e454fb5/coverage-7.10.6.tar.gz", hash = "sha256:f644a3ae5933a552a29dbb9aa2f90c677a875f80ebea028e5a52a4f429044b90", size = 823736, upload-time = "2025-08-29T15:35:16.668Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/1d/2e64b43d978b5bd184e0756a41415597dfef30fcbd90b747474bd749d45f/coverage-7.10.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70e7bfbd57126b5554aa482691145f798d7df77489a177a6bef80de78860a356", size = 217025, upload-time = "2025-08-29T15:32:57.169Z" }, + { url = "https://files.pythonhosted.org/packages/23/62/b1e0f513417c02cc10ef735c3ee5186df55f190f70498b3702d516aad06f/coverage-7.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e41be6f0f19da64af13403e52f2dec38bbc2937af54df8ecef10850ff8d35301", size = 217419, upload-time = "2025-08-29T15:32:59.908Z" }, + { url = "https://files.pythonhosted.org/packages/e7/16/b800640b7a43e7c538429e4d7223e0a94fd72453a1a048f70bf766f12e96/coverage-7.10.6-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c61fc91ab80b23f5fddbee342d19662f3d3328173229caded831aa0bd7595460", size = 244180, upload-time = "2025-08-29T15:33:01.608Z" }, + { url = "https://files.pythonhosted.org/packages/fb/6f/5e03631c3305cad187eaf76af0b559fff88af9a0b0c180d006fb02413d7a/coverage-7.10.6-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10356fdd33a7cc06e8051413140bbdc6f972137508a3572e3f59f805cd2832fd", size = 245992, upload-time = "2025-08-29T15:33:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a1/f30ea0fb400b080730125b490771ec62b3375789f90af0bb68bfb8a921d7/coverage-7.10.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80b1695cf7c5ebe7b44bf2521221b9bb8cdf69b1f24231149a7e3eb1ae5fa2fb", size = 247851, upload-time = "2025-08-29T15:33:04.603Z" }, + { url = "https://files.pythonhosted.org/packages/02/8e/cfa8fee8e8ef9a6bb76c7bef039f3302f44e615d2194161a21d3d83ac2e9/coverage-7.10.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2e4c33e6378b9d52d3454bd08847a8651f4ed23ddbb4a0520227bd346382bbc6", size = 245891, upload-time = "2025-08-29T15:33:06.176Z" }, + { url = "https://files.pythonhosted.org/packages/93/a9/51be09b75c55c4f6c16d8d73a6a1d46ad764acca0eab48fa2ffaef5958fe/coverage-7.10.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c8a3ec16e34ef980a46f60dc6ad86ec60f763c3f2fa0db6d261e6e754f72e945", size = 243909, upload-time = "2025-08-29T15:33:07.74Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a6/ba188b376529ce36483b2d585ca7bdac64aacbe5aa10da5978029a9c94db/coverage-7.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7d79dabc0a56f5af990cc6da9ad1e40766e82773c075f09cc571e2076fef882e", size = 244786, upload-time = "2025-08-29T15:33:08.965Z" }, + { url = "https://files.pythonhosted.org/packages/d0/4c/37ed872374a21813e0d3215256180c9a382c3f5ced6f2e5da0102fc2fd3e/coverage-7.10.6-cp310-cp310-win32.whl", hash = "sha256:86b9b59f2b16e981906e9d6383eb6446d5b46c278460ae2c36487667717eccf1", size = 219521, upload-time = "2025-08-29T15:33:10.599Z" }, + { url = "https://files.pythonhosted.org/packages/8e/36/9311352fdc551dec5b973b61f4e453227ce482985a9368305880af4f85dd/coverage-7.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:e132b9152749bd33534e5bd8565c7576f135f157b4029b975e15ee184325f528", size = 220417, upload-time = "2025-08-29T15:33:11.907Z" }, + { url = "https://files.pythonhosted.org/packages/d4/16/2bea27e212c4980753d6d563a0803c150edeaaddb0771a50d2afc410a261/coverage-7.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c706db3cabb7ceef779de68270150665e710b46d56372455cd741184f3868d8f", size = 217129, upload-time = "2025-08-29T15:33:13.575Z" }, + { url = "https://files.pythonhosted.org/packages/2a/51/e7159e068831ab37e31aac0969d47b8c5ee25b7d307b51e310ec34869315/coverage-7.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e0c38dc289e0508ef68ec95834cb5d2e96fdbe792eaccaa1bccac3966bbadcc", size = 217532, upload-time = "2025-08-29T15:33:14.872Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c0/246ccbea53d6099325d25cd208df94ea435cd55f0db38099dd721efc7a1f/coverage-7.10.6-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:752a3005a1ded28f2f3a6e8787e24f28d6abe176ca64677bcd8d53d6fe2ec08a", size = 247931, upload-time = "2025-08-29T15:33:16.142Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fb/7435ef8ab9b2594a6e3f58505cc30e98ae8b33265d844007737946c59389/coverage-7.10.6-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:689920ecfd60f992cafca4f5477d55720466ad2c7fa29bb56ac8d44a1ac2b47a", size = 249864, upload-time = "2025-08-29T15:33:17.434Z" }, + { url = "https://files.pythonhosted.org/packages/51/f8/d9d64e8da7bcddb094d511154824038833c81e3a039020a9d6539bf303e9/coverage-7.10.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ec98435796d2624d6905820a42f82149ee9fc4f2d45c2c5bc5a44481cc50db62", size = 251969, upload-time = "2025-08-29T15:33:18.822Z" }, + { url = "https://files.pythonhosted.org/packages/43/28/c43ba0ef19f446d6463c751315140d8f2a521e04c3e79e5c5fe211bfa430/coverage-7.10.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b37201ce4a458c7a758ecc4efa92fa8ed783c66e0fa3c42ae19fc454a0792153", size = 249659, upload-time = "2025-08-29T15:33:20.407Z" }, + { url = "https://files.pythonhosted.org/packages/79/3e/53635bd0b72beaacf265784508a0b386defc9ab7fad99ff95f79ce9db555/coverage-7.10.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2904271c80898663c810a6b067920a61dd8d38341244a3605bd31ab55250dad5", size = 247714, upload-time = "2025-08-29T15:33:21.751Z" }, + { url = "https://files.pythonhosted.org/packages/4c/55/0964aa87126624e8c159e32b0bc4e84edef78c89a1a4b924d28dd8265625/coverage-7.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5aea98383463d6e1fa4e95416d8de66f2d0cb588774ee20ae1b28df826bcb619", size = 248351, upload-time = "2025-08-29T15:33:23.105Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/6cfa9dc518c6c8e14a691c54e53a9433ba67336c760607e299bfcf520cb1/coverage-7.10.6-cp311-cp311-win32.whl", hash = "sha256:e3fb1fa01d3598002777dd259c0c2e6d9d5e10e7222976fc8e03992f972a2cba", size = 219562, upload-time = "2025-08-29T15:33:24.717Z" }, + { url = "https://files.pythonhosted.org/packages/5b/18/99b25346690cbc55922e7cfef06d755d4abee803ef335baff0014268eff4/coverage-7.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:f35ed9d945bece26553d5b4c8630453169672bea0050a564456eb88bdffd927e", size = 220453, upload-time = "2025-08-29T15:33:26.482Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ed/81d86648a07ccb124a5cf1f1a7788712b8d7216b593562683cd5c9b0d2c1/coverage-7.10.6-cp311-cp311-win_arm64.whl", hash = "sha256:99e1a305c7765631d74b98bf7dbf54eeea931f975e80f115437d23848ee8c27c", size = 219127, upload-time = "2025-08-29T15:33:27.777Z" }, + { url = "https://files.pythonhosted.org/packages/26/06/263f3305c97ad78aab066d116b52250dd316e74fcc20c197b61e07eb391a/coverage-7.10.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5b2dd6059938063a2c9fee1af729d4f2af28fd1a545e9b7652861f0d752ebcea", size = 217324, upload-time = "2025-08-29T15:33:29.06Z" }, + { url = "https://files.pythonhosted.org/packages/e9/60/1e1ded9a4fe80d843d7d53b3e395c1db3ff32d6c301e501f393b2e6c1c1f/coverage-7.10.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:388d80e56191bf846c485c14ae2bc8898aa3124d9d35903fef7d907780477634", size = 217560, upload-time = "2025-08-29T15:33:30.748Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/52136173c14e26dfed8b106ed725811bb53c30b896d04d28d74cb64318b3/coverage-7.10.6-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:90cb5b1a4670662719591aa92d0095bb41714970c0b065b02a2610172dbf0af6", size = 249053, upload-time = "2025-08-29T15:33:32.041Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1d/ae25a7dc58fcce8b172d42ffe5313fc267afe61c97fa872b80ee72d9515a/coverage-7.10.6-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:961834e2f2b863a0e14260a9a273aff07ff7818ab6e66d2addf5628590c628f9", size = 251802, upload-time = "2025-08-29T15:33:33.625Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7a/1f561d47743710fe996957ed7c124b421320f150f1d38523d8d9102d3e2a/coverage-7.10.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf9a19f5012dab774628491659646335b1928cfc931bf8d97b0d5918dd58033c", size = 252935, upload-time = "2025-08-29T15:33:34.909Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ad/8b97cd5d28aecdfde792dcbf646bac141167a5cacae2cd775998b45fabb5/coverage-7.10.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:99c4283e2a0e147b9c9cc6bc9c96124de9419d6044837e9799763a0e29a7321a", size = 250855, upload-time = "2025-08-29T15:33:36.922Z" }, + { url = "https://files.pythonhosted.org/packages/33/6a/95c32b558d9a61858ff9d79580d3877df3eb5bc9eed0941b1f187c89e143/coverage-7.10.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:282b1b20f45df57cc508c1e033403f02283adfb67d4c9c35a90281d81e5c52c5", size = 248974, upload-time = "2025-08-29T15:33:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/8ce95dee640a38e760d5b747c10913e7a06554704d60b41e73fdea6a1ffd/coverage-7.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8cdbe264f11afd69841bd8c0d83ca10b5b32853263ee62e6ac6a0ab63895f972", size = 250409, upload-time = "2025-08-29T15:33:39.447Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/7a55b0bdde78a98e2eb2356771fd2dcddb96579e8342bb52aa5bc52e96f0/coverage-7.10.6-cp312-cp312-win32.whl", hash = "sha256:a517feaf3a0a3eca1ee985d8373135cfdedfbba3882a5eab4362bda7c7cf518d", size = 219724, upload-time = "2025-08-29T15:33:41.172Z" }, + { url = "https://files.pythonhosted.org/packages/36/4a/32b185b8b8e327802c9efce3d3108d2fe2d9d31f153a0f7ecfd59c773705/coverage-7.10.6-cp312-cp312-win_amd64.whl", hash = "sha256:856986eadf41f52b214176d894a7de05331117f6035a28ac0016c0f63d887629", size = 220536, upload-time = "2025-08-29T15:33:42.524Z" }, + { url = "https://files.pythonhosted.org/packages/08/3a/d5d8dc703e4998038c3099eaf77adddb00536a3cec08c8dcd556a36a3eb4/coverage-7.10.6-cp312-cp312-win_arm64.whl", hash = "sha256:acf36b8268785aad739443fa2780c16260ee3fa09d12b3a70f772ef100939d80", size = 219171, upload-time = "2025-08-29T15:33:43.974Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e7/917e5953ea29a28c1057729c1d5af9084ab6d9c66217523fd0e10f14d8f6/coverage-7.10.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ffea0575345e9ee0144dfe5701aa17f3ba546f8c3bb48db62ae101afb740e7d6", size = 217351, upload-time = "2025-08-29T15:33:45.438Z" }, + { url = "https://files.pythonhosted.org/packages/eb/86/2e161b93a4f11d0ea93f9bebb6a53f113d5d6e416d7561ca41bb0a29996b/coverage-7.10.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:95d91d7317cde40a1c249d6b7382750b7e6d86fad9d8eaf4fa3f8f44cf171e80", size = 217600, upload-time = "2025-08-29T15:33:47.269Z" }, + { url = "https://files.pythonhosted.org/packages/0e/66/d03348fdd8df262b3a7fb4ee5727e6e4936e39e2f3a842e803196946f200/coverage-7.10.6-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3e23dd5408fe71a356b41baa82892772a4cefcf758f2ca3383d2aa39e1b7a003", size = 248600, upload-time = "2025-08-29T15:33:48.953Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/508420fb47d09d904d962f123221bc249f64b5e56aa93d5f5f7603be475f/coverage-7.10.6-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0f3f56e4cb573755e96a16501a98bf211f100463d70275759e73f3cbc00d4f27", size = 251206, upload-time = "2025-08-29T15:33:50.697Z" }, + { url = "https://files.pythonhosted.org/packages/e9/1f/9020135734184f439da85c70ea78194c2730e56c2d18aee6e8ff1719d50d/coverage-7.10.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db4a1d897bbbe7339946ffa2fe60c10cc81c43fab8b062d3fcb84188688174a4", size = 252478, upload-time = "2025-08-29T15:33:52.303Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a4/3d228f3942bb5a2051fde28c136eea23a761177dc4ff4ef54533164ce255/coverage-7.10.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d8fd7879082953c156d5b13c74aa6cca37f6a6f4747b39538504c3f9c63d043d", size = 250637, upload-time = "2025-08-29T15:33:53.67Z" }, + { url = "https://files.pythonhosted.org/packages/36/e3/293dce8cdb9a83de971637afc59b7190faad60603b40e32635cbd15fbf61/coverage-7.10.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:28395ca3f71cd103b8c116333fa9db867f3a3e1ad6a084aa3725ae002b6583bc", size = 248529, upload-time = "2025-08-29T15:33:55.022Z" }, + { url = "https://files.pythonhosted.org/packages/90/26/64eecfa214e80dd1d101e420cab2901827de0e49631d666543d0e53cf597/coverage-7.10.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:61c950fc33d29c91b9e18540e1aed7d9f6787cc870a3e4032493bbbe641d12fc", size = 250143, upload-time = "2025-08-29T15:33:56.386Z" }, + { url = "https://files.pythonhosted.org/packages/3e/70/bd80588338f65ea5b0d97e424b820fb4068b9cfb9597fbd91963086e004b/coverage-7.10.6-cp313-cp313-win32.whl", hash = "sha256:160c00a5e6b6bdf4e5984b0ef21fc860bc94416c41b7df4d63f536d17c38902e", size = 219770, upload-time = "2025-08-29T15:33:58.063Z" }, + { url = "https://files.pythonhosted.org/packages/a7/14/0b831122305abcc1060c008f6c97bbdc0a913ab47d65070a01dc50293c2b/coverage-7.10.6-cp313-cp313-win_amd64.whl", hash = "sha256:628055297f3e2aa181464c3808402887643405573eb3d9de060d81531fa79d32", size = 220566, upload-time = "2025-08-29T15:33:59.766Z" }, + { url = "https://files.pythonhosted.org/packages/83/c6/81a83778c1f83f1a4a168ed6673eeedc205afb562d8500175292ca64b94e/coverage-7.10.6-cp313-cp313-win_arm64.whl", hash = "sha256:df4ec1f8540b0bcbe26ca7dd0f541847cc8a108b35596f9f91f59f0c060bfdd2", size = 219195, upload-time = "2025-08-29T15:34:01.191Z" }, + { url = "https://files.pythonhosted.org/packages/d7/1c/ccccf4bf116f9517275fa85047495515add43e41dfe8e0bef6e333c6b344/coverage-7.10.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:c9a8b7a34a4de3ed987f636f71881cd3b8339f61118b1aa311fbda12741bff0b", size = 218059, upload-time = "2025-08-29T15:34:02.91Z" }, + { url = "https://files.pythonhosted.org/packages/92/97/8a3ceff833d27c7492af4f39d5da6761e9ff624831db9e9f25b3886ddbca/coverage-7.10.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dd5af36092430c2b075cee966719898f2ae87b636cefb85a653f1d0ba5d5393", size = 218287, upload-time = "2025-08-29T15:34:05.106Z" }, + { url = "https://files.pythonhosted.org/packages/92/d8/50b4a32580cf41ff0423777a2791aaf3269ab60c840b62009aec12d3970d/coverage-7.10.6-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b0353b0f0850d49ada66fdd7d0c7cdb0f86b900bb9e367024fd14a60cecc1e27", size = 259625, upload-time = "2025-08-29T15:34:06.575Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7e/6a7df5a6fb440a0179d94a348eb6616ed4745e7df26bf2a02bc4db72c421/coverage-7.10.6-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d6b9ae13d5d3e8aeca9ca94198aa7b3ebbc5acfada557d724f2a1f03d2c0b0df", size = 261801, upload-time = "2025-08-29T15:34:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/3a/4c/a270a414f4ed5d196b9d3d67922968e768cd971d1b251e1b4f75e9362f75/coverage-7.10.6-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:675824a363cc05781b1527b39dc2587b8984965834a748177ee3c37b64ffeafb", size = 264027, upload-time = "2025-08-29T15:34:09.806Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8b/3210d663d594926c12f373c5370bf1e7c5c3a427519a8afa65b561b9a55c/coverage-7.10.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:692d70ea725f471a547c305f0d0fc6a73480c62fb0da726370c088ab21aed282", size = 261576, upload-time = "2025-08-29T15:34:11.585Z" }, + { url = "https://files.pythonhosted.org/packages/72/d0/e1961eff67e9e1dba3fc5eb7a4caf726b35a5b03776892da8d79ec895775/coverage-7.10.6-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:851430a9a361c7a8484a36126d1d0ff8d529d97385eacc8dfdc9bfc8c2d2cbe4", size = 259341, upload-time = "2025-08-29T15:34:13.159Z" }, + { url = "https://files.pythonhosted.org/packages/3a/06/d6478d152cd189b33eac691cba27a40704990ba95de49771285f34a5861e/coverage-7.10.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d9369a23186d189b2fc95cc08b8160ba242057e887d766864f7adf3c46b2df21", size = 260468, upload-time = "2025-08-29T15:34:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/ed/73/737440247c914a332f0b47f7598535b29965bf305e19bbc22d4c39615d2b/coverage-7.10.6-cp313-cp313t-win32.whl", hash = "sha256:92be86fcb125e9bda0da7806afd29a3fd33fdf58fba5d60318399adf40bf37d0", size = 220429, upload-time = "2025-08-29T15:34:16.394Z" }, + { url = "https://files.pythonhosted.org/packages/bd/76/b92d3214740f2357ef4a27c75a526eb6c28f79c402e9f20a922c295c05e2/coverage-7.10.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6b3039e2ca459a70c79523d39347d83b73f2f06af5624905eba7ec34d64d80b5", size = 221493, upload-time = "2025-08-29T15:34:17.835Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8e/6dcb29c599c8a1f654ec6cb68d76644fe635513af16e932d2d4ad1e5ac6e/coverage-7.10.6-cp313-cp313t-win_arm64.whl", hash = "sha256:3fb99d0786fe17b228eab663d16bee2288e8724d26a199c29325aac4b0319b9b", size = 219757, upload-time = "2025-08-29T15:34:19.248Z" }, + { url = "https://files.pythonhosted.org/packages/d3/aa/76cf0b5ec00619ef208da4689281d48b57f2c7fde883d14bf9441b74d59f/coverage-7.10.6-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6008a021907be8c4c02f37cdc3ffb258493bdebfeaf9a839f9e71dfdc47b018e", size = 217331, upload-time = "2025-08-29T15:34:20.846Z" }, + { url = "https://files.pythonhosted.org/packages/65/91/8e41b8c7c505d398d7730206f3cbb4a875a35ca1041efc518051bfce0f6b/coverage-7.10.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5e75e37f23eb144e78940b40395b42f2321951206a4f50e23cfd6e8a198d3ceb", size = 217607, upload-time = "2025-08-29T15:34:22.433Z" }, + { url = "https://files.pythonhosted.org/packages/87/7f/f718e732a423d442e6616580a951b8d1ec3575ea48bcd0e2228386805e79/coverage-7.10.6-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0f7cb359a448e043c576f0da00aa8bfd796a01b06aa610ca453d4dde09cc1034", size = 248663, upload-time = "2025-08-29T15:34:24.425Z" }, + { url = "https://files.pythonhosted.org/packages/e6/52/c1106120e6d801ac03e12b5285e971e758e925b6f82ee9b86db3aa10045d/coverage-7.10.6-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c68018e4fc4e14b5668f1353b41ccf4bc83ba355f0e1b3836861c6f042d89ac1", size = 251197, upload-time = "2025-08-29T15:34:25.906Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ec/3a8645b1bb40e36acde9c0609f08942852a4af91a937fe2c129a38f2d3f5/coverage-7.10.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cd4b2b0707fc55afa160cd5fc33b27ccbf75ca11d81f4ec9863d5793fc6df56a", size = 252551, upload-time = "2025-08-29T15:34:27.337Z" }, + { url = "https://files.pythonhosted.org/packages/a1/70/09ecb68eeb1155b28a1d16525fd3a9b65fbe75337311a99830df935d62b6/coverage-7.10.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4cec13817a651f8804a86e4f79d815b3b28472c910e099e4d5a0e8a3b6a1d4cb", size = 250553, upload-time = "2025-08-29T15:34:29.065Z" }, + { url = "https://files.pythonhosted.org/packages/c6/80/47df374b893fa812e953b5bc93dcb1427a7b3d7a1a7d2db33043d17f74b9/coverage-7.10.6-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f2a6a8e06bbda06f78739f40bfb56c45d14eb8249d0f0ea6d4b3d48e1f7c695d", size = 248486, upload-time = "2025-08-29T15:34:30.897Z" }, + { url = "https://files.pythonhosted.org/packages/4a/65/9f98640979ecee1b0d1a7164b589de720ddf8100d1747d9bbdb84be0c0fb/coverage-7.10.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:081b98395ced0d9bcf60ada7661a0b75f36b78b9d7e39ea0790bb4ed8da14747", size = 249981, upload-time = "2025-08-29T15:34:32.365Z" }, + { url = "https://files.pythonhosted.org/packages/1f/55/eeb6603371e6629037f47bd25bef300387257ed53a3c5fdb159b7ac8c651/coverage-7.10.6-cp314-cp314-win32.whl", hash = "sha256:6937347c5d7d069ee776b2bf4e1212f912a9f1f141a429c475e6089462fcecc5", size = 220054, upload-time = "2025-08-29T15:34:34.124Z" }, + { url = "https://files.pythonhosted.org/packages/15/d1/a0912b7611bc35412e919a2cd59ae98e7ea3b475e562668040a43fb27897/coverage-7.10.6-cp314-cp314-win_amd64.whl", hash = "sha256:adec1d980fa07e60b6ef865f9e5410ba760e4e1d26f60f7e5772c73b9a5b0713", size = 220851, upload-time = "2025-08-29T15:34:35.651Z" }, + { url = "https://files.pythonhosted.org/packages/ef/2d/11880bb8ef80a45338e0b3e0725e4c2d73ffbb4822c29d987078224fd6a5/coverage-7.10.6-cp314-cp314-win_arm64.whl", hash = "sha256:a80f7aef9535442bdcf562e5a0d5a5538ce8abe6bb209cfbf170c462ac2c2a32", size = 219429, upload-time = "2025-08-29T15:34:37.16Z" }, + { url = "https://files.pythonhosted.org/packages/83/c0/1f00caad775c03a700146f55536ecd097a881ff08d310a58b353a1421be0/coverage-7.10.6-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:0de434f4fbbe5af4fa7989521c655c8c779afb61c53ab561b64dcee6149e4c65", size = 218080, upload-time = "2025-08-29T15:34:38.919Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c4/b1c5d2bd7cc412cbeb035e257fd06ed4e3e139ac871d16a07434e145d18d/coverage-7.10.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e31b8155150c57e5ac43ccd289d079eb3f825187d7c66e755a055d2c85794c6", size = 218293, upload-time = "2025-08-29T15:34:40.425Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/4468d37c94724bf6ec354e4ec2f205fda194343e3e85fd2e59cec57e6a54/coverage-7.10.6-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:98cede73eb83c31e2118ae8d379c12e3e42736903a8afcca92a7218e1f2903b0", size = 259800, upload-time = "2025-08-29T15:34:41.996Z" }, + { url = "https://files.pythonhosted.org/packages/82/d8/f8fb351be5fee31690cd8da768fd62f1cfab33c31d9f7baba6cd8960f6b8/coverage-7.10.6-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f863c08f4ff6b64fa8045b1e3da480f5374779ef187f07b82e0538c68cb4ff8e", size = 261965, upload-time = "2025-08-29T15:34:43.61Z" }, + { url = "https://files.pythonhosted.org/packages/e8/70/65d4d7cfc75c5c6eb2fed3ee5cdf420fd8ae09c4808723a89a81d5b1b9c3/coverage-7.10.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b38261034fda87be356f2c3f42221fdb4171c3ce7658066ae449241485390d5", size = 264220, upload-time = "2025-08-29T15:34:45.387Z" }, + { url = "https://files.pythonhosted.org/packages/98/3c/069df106d19024324cde10e4ec379fe2fb978017d25e97ebee23002fbadf/coverage-7.10.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e93b1476b79eae849dc3872faeb0bf7948fd9ea34869590bc16a2a00b9c82a7", size = 261660, upload-time = "2025-08-29T15:34:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/2974d53904080c5dc91af798b3a54a4ccb99a45595cc0dcec6eb9616a57d/coverage-7.10.6-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ff8a991f70f4c0cf53088abf1e3886edcc87d53004c7bb94e78650b4d3dac3b5", size = 259417, upload-time = "2025-08-29T15:34:48.779Z" }, + { url = "https://files.pythonhosted.org/packages/30/38/9616a6b49c686394b318974d7f6e08f38b8af2270ce7488e879888d1e5db/coverage-7.10.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ac765b026c9f33044419cbba1da913cfb82cca1b60598ac1c7a5ed6aac4621a0", size = 260567, upload-time = "2025-08-29T15:34:50.718Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/3ed2d6312b371a8cf804abf4e14895b70e4c3491c6e53536d63fd0958a8d/coverage-7.10.6-cp314-cp314t-win32.whl", hash = "sha256:441c357d55f4936875636ef2cfb3bee36e466dcf50df9afbd398ce79dba1ebb7", size = 220831, upload-time = "2025-08-29T15:34:52.653Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e5/d38d0cb830abede2adb8b147770d2a3d0e7fecc7228245b9b1ae6c24930a/coverage-7.10.6-cp314-cp314t-win_amd64.whl", hash = "sha256:073711de3181b2e204e4870ac83a7c4853115b42e9cd4d145f2231e12d670930", size = 221950, upload-time = "2025-08-29T15:34:54.212Z" }, + { url = "https://files.pythonhosted.org/packages/f4/51/e48e550f6279349895b0ffcd6d2a690e3131ba3a7f4eafccc141966d4dea/coverage-7.10.6-cp314-cp314t-win_arm64.whl", hash = "sha256:137921f2bac5559334ba66122b753db6dc5d1cf01eb7b64eb412bb0d064ef35b", size = 219969, upload-time = "2025-08-29T15:34:55.83Z" }, + { url = "https://files.pythonhosted.org/packages/44/0c/50db5379b615854b5cf89146f8f5bd1d5a9693d7f3a987e269693521c404/coverage-7.10.6-py3-none-any.whl", hash = "sha256:92c4ecf6bf11b2e85fd4d8204814dc26e6a19f0c9d938c207c5cb0eadfcabbe3", size = 208986, upload-time = "2025-08-29T15:35:14.506Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "cryptography" +version = "45.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/35/c495bffc2056f2dadb32434f1feedd79abde2a7f8363e1974afa9c33c7e2/cryptography-45.0.7.tar.gz", hash = "sha256:4b1654dfc64ea479c242508eb8c724044f1e964a47d1d1cacc5132292d851971", size = 744980, upload-time = "2025-09-01T11:15:03.146Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/91/925c0ac74362172ae4516000fe877912e33b5983df735ff290c653de4913/cryptography-45.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3be4f21c6245930688bd9e162829480de027f8bf962ede33d4f8ba7d67a00cee", size = 7041105, upload-time = "2025-09-01T11:13:59.684Z" }, + { url = "https://files.pythonhosted.org/packages/fc/63/43641c5acce3a6105cf8bd5baeceeb1846bb63067d26dae3e5db59f1513a/cryptography-45.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67285f8a611b0ebc0857ced2081e30302909f571a46bfa7a3cc0ad303fe015c6", size = 4205799, upload-time = "2025-09-01T11:14:02.517Z" }, + { url = "https://files.pythonhosted.org/packages/bc/29/c238dd9107f10bfde09a4d1c52fd38828b1aa353ced11f358b5dd2507d24/cryptography-45.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:577470e39e60a6cd7780793202e63536026d9b8641de011ed9d8174da9ca5339", size = 4430504, upload-time = "2025-09-01T11:14:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/62/62/24203e7cbcc9bd7c94739428cd30680b18ae6b18377ae66075c8e4771b1b/cryptography-45.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4bd3e5c4b9682bc112d634f2c6ccc6736ed3635fc3319ac2bb11d768cc5a00d8", size = 4209542, upload-time = "2025-09-01T11:14:06.309Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e3/e7de4771a08620eef2389b86cd87a2c50326827dea5528feb70595439ce4/cryptography-45.0.7-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:465ccac9d70115cd4de7186e60cfe989de73f7bb23e8a7aa45af18f7412e75bf", size = 3889244, upload-time = "2025-09-01T11:14:08.152Z" }, + { url = "https://files.pythonhosted.org/packages/96/b8/bca71059e79a0bb2f8e4ec61d9c205fbe97876318566cde3b5092529faa9/cryptography-45.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:16ede8a4f7929b4b7ff3642eba2bf79aa1d71f24ab6ee443935c0d269b6bc513", size = 4461975, upload-time = "2025-09-01T11:14:09.755Z" }, + { url = "https://files.pythonhosted.org/packages/58/67/3f5b26937fe1218c40e95ef4ff8d23c8dc05aa950d54200cc7ea5fb58d28/cryptography-45.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8978132287a9d3ad6b54fcd1e08548033cc09dc6aacacb6c004c73c3eb5d3ac3", size = 4209082, upload-time = "2025-09-01T11:14:11.229Z" }, + { url = "https://files.pythonhosted.org/packages/0e/e4/b3e68a4ac363406a56cf7b741eeb80d05284d8c60ee1a55cdc7587e2a553/cryptography-45.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b6a0e535baec27b528cb07a119f321ac024592388c5681a5ced167ae98e9fff3", size = 4460397, upload-time = "2025-09-01T11:14:12.924Z" }, + { url = "https://files.pythonhosted.org/packages/22/49/2c93f3cd4e3efc8cb22b02678c1fad691cff9dd71bb889e030d100acbfe0/cryptography-45.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a24ee598d10befaec178efdff6054bc4d7e883f615bfbcd08126a0f4931c83a6", size = 4337244, upload-time = "2025-09-01T11:14:14.431Z" }, + { url = "https://files.pythonhosted.org/packages/04/19/030f400de0bccccc09aa262706d90f2ec23d56bc4eb4f4e8268d0ddf3fb8/cryptography-45.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa26fa54c0a9384c27fcdc905a2fb7d60ac6e47d14bc2692145f2b3b1e2cfdbd", size = 4568862, upload-time = "2025-09-01T11:14:16.185Z" }, + { url = "https://files.pythonhosted.org/packages/29/56/3034a3a353efa65116fa20eb3c990a8c9f0d3db4085429040a7eef9ada5f/cryptography-45.0.7-cp311-abi3-win32.whl", hash = "sha256:bef32a5e327bd8e5af915d3416ffefdbe65ed975b646b3805be81b23580b57b8", size = 2936578, upload-time = "2025-09-01T11:14:17.638Z" }, + { url = "https://files.pythonhosted.org/packages/b3/61/0ab90f421c6194705a99d0fa9f6ee2045d916e4455fdbb095a9c2c9a520f/cryptography-45.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:3808e6b2e5f0b46d981c24d79648e5c25c35e59902ea4391a0dcb3e667bf7443", size = 3405400, upload-time = "2025-09-01T11:14:18.958Z" }, + { url = "https://files.pythonhosted.org/packages/63/e8/c436233ddf19c5f15b25ace33979a9dd2e7aa1a59209a0ee8554179f1cc0/cryptography-45.0.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bfb4c801f65dd61cedfc61a83732327fafbac55a47282e6f26f073ca7a41c3b2", size = 7021824, upload-time = "2025-09-01T11:14:20.954Z" }, + { url = "https://files.pythonhosted.org/packages/bc/4c/8f57f2500d0ccd2675c5d0cc462095adf3faa8c52294ba085c036befb901/cryptography-45.0.7-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:81823935e2f8d476707e85a78a405953a03ef7b7b4f55f93f7c2d9680e5e0691", size = 4202233, upload-time = "2025-09-01T11:14:22.454Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ac/59b7790b4ccaed739fc44775ce4645c9b8ce54cbec53edf16c74fd80cb2b/cryptography-45.0.7-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3994c809c17fc570c2af12c9b840d7cea85a9fd3e5c0e0491f4fa3c029216d59", size = 4423075, upload-time = "2025-09-01T11:14:24.287Z" }, + { url = "https://files.pythonhosted.org/packages/b8/56/d4f07ea21434bf891faa088a6ac15d6d98093a66e75e30ad08e88aa2b9ba/cryptography-45.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dad43797959a74103cb59c5dac71409f9c27d34c8a05921341fb64ea8ccb1dd4", size = 4204517, upload-time = "2025-09-01T11:14:25.679Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ac/924a723299848b4c741c1059752c7cfe09473b6fd77d2920398fc26bfb53/cryptography-45.0.7-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ce7a453385e4c4693985b4a4a3533e041558851eae061a58a5405363b098fcd3", size = 3882893, upload-time = "2025-09-01T11:14:27.1Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/4dab2ff0a871cc2d81d3ae6d780991c0192b259c35e4d83fe1de18b20c70/cryptography-45.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b04f85ac3a90c227b6e5890acb0edbaf3140938dbecf07bff618bf3638578cf1", size = 4450132, upload-time = "2025-09-01T11:14:28.58Z" }, + { url = "https://files.pythonhosted.org/packages/12/dd/b2882b65db8fc944585d7fb00d67cf84a9cef4e77d9ba8f69082e911d0de/cryptography-45.0.7-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:48c41a44ef8b8c2e80ca4527ee81daa4c527df3ecbc9423c41a420a9559d0e27", size = 4204086, upload-time = "2025-09-01T11:14:30.572Z" }, + { url = "https://files.pythonhosted.org/packages/5d/fa/1d5745d878048699b8eb87c984d4ccc5da4f5008dfd3ad7a94040caca23a/cryptography-45.0.7-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f3df7b3d0f91b88b2106031fd995802a2e9ae13e02c36c1fc075b43f420f3a17", size = 4449383, upload-time = "2025-09-01T11:14:32.046Z" }, + { url = "https://files.pythonhosted.org/packages/36/8b/fc61f87931bc030598e1876c45b936867bb72777eac693e905ab89832670/cryptography-45.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dd342f085542f6eb894ca00ef70236ea46070c8a13824c6bde0dfdcd36065b9b", size = 4332186, upload-time = "2025-09-01T11:14:33.95Z" }, + { url = "https://files.pythonhosted.org/packages/0b/11/09700ddad7443ccb11d674efdbe9a832b4455dc1f16566d9bd3834922ce5/cryptography-45.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1993a1bb7e4eccfb922b6cd414f072e08ff5816702a0bdb8941c247a6b1b287c", size = 4561639, upload-time = "2025-09-01T11:14:35.343Z" }, + { url = "https://files.pythonhosted.org/packages/71/ed/8f4c1337e9d3b94d8e50ae0b08ad0304a5709d483bfcadfcc77a23dbcb52/cryptography-45.0.7-cp37-abi3-win32.whl", hash = "sha256:18fcf70f243fe07252dcb1b268a687f2358025ce32f9f88028ca5c364b123ef5", size = 2926552, upload-time = "2025-09-01T11:14:36.929Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ff/026513ecad58dacd45d1d24ebe52b852165a26e287177de1d545325c0c25/cryptography-45.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:7285a89df4900ed3bfaad5679b1e668cb4b38a8de1ccbfc84b05f34512da0a90", size = 3392742, upload-time = "2025-09-01T11:14:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/13/3e/e42f1528ca1ea82256b835191eab1be014e0f9f934b60d98b0be8a38ed70/cryptography-45.0.7-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:de58755d723e86175756f463f2f0bddd45cc36fbd62601228a3f8761c9f58252", size = 3572442, upload-time = "2025-09-01T11:14:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/59/aa/e947693ab08674a2663ed2534cd8d345cf17bf6a1facf99273e8ec8986dc/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a20e442e917889d1a6b3c570c9e3fa2fdc398c20868abcea268ea33c024c4083", size = 4142233, upload-time = "2025-09-01T11:14:41.305Z" }, + { url = "https://files.pythonhosted.org/packages/24/06/09b6f6a2fc43474a32b8fe259038eef1500ee3d3c141599b57ac6c57612c/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:258e0dff86d1d891169b5af222d362468a9570e2532923088658aa866eb11130", size = 4376202, upload-time = "2025-09-01T11:14:43.047Z" }, + { url = "https://files.pythonhosted.org/packages/00/f2/c166af87e95ce6ae6d38471a7e039d3a0549c2d55d74e059680162052824/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d97cf502abe2ab9eff8bd5e4aca274da8d06dd3ef08b759a8d6143f4ad65d4b4", size = 4141900, upload-time = "2025-09-01T11:14:45.089Z" }, + { url = "https://files.pythonhosted.org/packages/16/b9/e96e0b6cb86eae27ea51fa8a3151535a18e66fe7c451fa90f7f89c85f541/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:c987dad82e8c65ebc985f5dae5e74a3beda9d0a2a4daf8a1115f3772b59e5141", size = 4375562, upload-time = "2025-09-01T11:14:47.166Z" }, + { url = "https://files.pythonhosted.org/packages/36/d0/36e8ee39274e9d77baf7d0dafda680cba6e52f3936b846f0d56d64fec915/cryptography-45.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c13b1e3afd29a5b3b2656257f14669ca8fa8d7956d509926f0b130b600b50ab7", size = 3322781, upload-time = "2025-09-01T11:14:48.747Z" }, + { url = "https://files.pythonhosted.org/packages/99/4e/49199a4c82946938a3e05d2e8ad9482484ba48bbc1e809e3d506c686d051/cryptography-45.0.7-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a862753b36620af6fc54209264f92c716367f2f0ff4624952276a6bbd18cbde", size = 3584634, upload-time = "2025-09-01T11:14:50.593Z" }, + { url = "https://files.pythonhosted.org/packages/16/ce/5f6ff59ea9c7779dba51b84871c19962529bdcc12e1a6ea172664916c550/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:06ce84dc14df0bf6ea84666f958e6080cdb6fe1231be2a51f3fc1267d9f3fb34", size = 4149533, upload-time = "2025-09-01T11:14:52.091Z" }, + { url = "https://files.pythonhosted.org/packages/ce/13/b3cfbd257ac96da4b88b46372e662009b7a16833bfc5da33bb97dd5631ae/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d0c5c6bac22b177bf8da7435d9d27a6834ee130309749d162b26c3105c0795a9", size = 4385557, upload-time = "2025-09-01T11:14:53.551Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c5/8c59d6b7c7b439ba4fc8d0cab868027fd095f215031bc123c3a070962912/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:2f641b64acc00811da98df63df7d59fd4706c0df449da71cb7ac39a0732b40ae", size = 4149023, upload-time = "2025-09-01T11:14:55.022Z" }, + { url = "https://files.pythonhosted.org/packages/55/32/05385c86d6ca9ab0b4d5bb442d2e3d85e727939a11f3e163fc776ce5eb40/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:f5414a788ecc6ee6bc58560e85ca624258a55ca434884445440a810796ea0e0b", size = 4385722, upload-time = "2025-09-01T11:14:57.319Z" }, + { url = "https://files.pythonhosted.org/packages/23/87/7ce86f3fa14bc11a5a48c30d8103c26e09b6465f8d8e9d74cf7a0714f043/cryptography-45.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1f3d56f73595376f4244646dd5c5870c14c196949807be39e79e7bd9bac3da63", size = 3332908, upload-time = "2025-09-01T11:14:58.78Z" }, +] + +[[package]] +name = "ec2-gha" +version = "1.0.0" +source = { editable = "." } +dependencies = [ + { name = "boto3" }, + { name = "gha-runner" }, +] + +[package.optional-dependencies] +test = [ + { name = "moto" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "responses" }, + { name = "ruff" }, + { name = "syrupy" }, +] + +[package.metadata] +requires-dist = [ + { name = "boto3" }, + { name = "gha-runner", git = "https://github.com/Open-Athena/gha-runner.git?rev=v1" }, + { name = "moto", extras = ["ec2"], marker = "extra == 'test'" }, + { name = "pytest", marker = "extra == 'test'" }, + { name = "pytest-cov", marker = "extra == 'test'" }, + { name = "responses", marker = "extra == 'test'" }, + { name = "ruff", marker = "extra == 'test'" }, + { name = "syrupy", marker = "extra == 'test'" }, +] +provides-extras = ["test"] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "gha-runner" +version = "0.6.1.post19+ge8ef9e3" +source = { git = "https://github.com/Open-Athena/gha-runner.git?rev=v1#e8ef9e362e00d5be5a58a9af01091d784611bc05" } +dependencies = [ + { name = "requests" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357, upload-time = "2024-10-18T15:20:51.44Z" }, + { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393, upload-time = "2024-10-18T15:20:52.426Z" }, + { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732, upload-time = "2024-10-18T15:20:53.578Z" }, + { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866, upload-time = "2024-10-18T15:20:55.06Z" }, + { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964, upload-time = "2024-10-18T15:20:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977, upload-time = "2024-10-18T15:20:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366, upload-time = "2024-10-18T15:20:58.235Z" }, + { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091, upload-time = "2024-10-18T15:20:59.235Z" }, + { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065, upload-time = "2024-10-18T15:21:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514, upload-time = "2024-10-18T15:21:01.122Z" }, + { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" }, + { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" }, + { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" }, + { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" }, + { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274, upload-time = "2024-10-18T15:21:24.577Z" }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352, upload-time = "2024-10-18T15:21:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122, upload-time = "2024-10-18T15:21:26.199Z" }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085, upload-time = "2024-10-18T15:21:27.029Z" }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978, upload-time = "2024-10-18T15:21:27.846Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208, upload-time = "2024-10-18T15:21:28.744Z" }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357, upload-time = "2024-10-18T15:21:29.545Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344, upload-time = "2024-10-18T15:21:30.366Z" }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101, upload-time = "2024-10-18T15:21:31.207Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603, upload-time = "2024-10-18T15:21:32.032Z" }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510, upload-time = "2024-10-18T15:21:33.625Z" }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486, upload-time = "2024-10-18T15:21:34.611Z" }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480, upload-time = "2024-10-18T15:21:35.398Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914, upload-time = "2024-10-18T15:21:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796, upload-time = "2024-10-18T15:21:37.073Z" }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473, upload-time = "2024-10-18T15:21:37.932Z" }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114, upload-time = "2024-10-18T15:21:39.799Z" }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098, upload-time = "2024-10-18T15:21:40.813Z" }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208, upload-time = "2024-10-18T15:21:41.814Z" }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739, upload-time = "2024-10-18T15:21:42.784Z" }, +] + +[[package]] +name = "moto" +version = "5.1.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "cryptography" }, + { name = "jinja2" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "responses" }, + { name = "werkzeug" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/71/805c0a0b30e362cd759206d4723bad800bc8c6c83a7edd05e55747d03959/moto-5.1.12.tar.gz", hash = "sha256:6eca3a020cb89c188b763610c27c969c32b832205712d3bdcb1a6031a4005187", size = 7185928, upload-time = "2025-09-07T19:38:37.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/92/cfb9a8be3d6070bef53afb92e03a5a7eb6da0127b29a8438fe00b12f5ed2/moto-5.1.12-py3-none-any.whl", hash = "sha256:c9f1119ab57819ce4b88f793f51c6ca0361b6932a90c59865fd71022acfc5582", size = 5313196, upload-time = "2025-09-07T19:38:34.78Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-cov" +version = "6.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/4c/f883ab8f0daad69f47efdf95f55a66b51a8b939c430dadce0611508d9e99/pytest_cov-6.3.0.tar.gz", hash = "sha256:35c580e7800f87ce892e687461166e1ac2bcb8fb9e13aea79032518d6e503ff2", size = 70398, upload-time = "2025-09-06T15:40:14.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/b4/bb7263e12aade3842b938bc5c6958cae79c5ee18992f9b9349019579da0f/pytest_cov-6.3.0-py3-none-any.whl", hash = "sha256:440db28156d2468cafc0415b4f8e50856a0d11faefa38f30906048fe490f1749", size = 25115, upload-time = "2025-09-06T15:40:12.44Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "responses" +version = "0.25.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/95/89c054ad70bfef6da605338b009b2e283485835351a9935c7bfbfaca7ffc/responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4", size = 79320, upload-time = "2025-08-08T19:01:46.709Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/4c/cc276ce57e572c102d9542d383b2cfd551276581dc60004cb94fe8774c11/responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c", size = 34769, upload-time = "2025-08-08T19:01:45.018Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/f0/e0965dd709b8cabe6356811c0ee8c096806bb57d20b5019eb4e48a117410/ruff-0.12.12.tar.gz", hash = "sha256:b86cd3415dbe31b3b46a71c598f4c4b2f550346d1ccf6326b347cc0c8fd063d6", size = 5359915, upload-time = "2025-09-04T16:50:18.273Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/79/8d3d687224d88367b51c7974cec1040c4b015772bfbeffac95face14c04a/ruff-0.12.12-py3-none-linux_armv6l.whl", hash = "sha256:de1c4b916d98ab289818e55ce481e2cacfaad7710b01d1f990c497edf217dafc", size = 12116602, upload-time = "2025-09-04T16:49:18.892Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c3/6e599657fe192462f94861a09aae935b869aea8a1da07f47d6eae471397c/ruff-0.12.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7acd6045e87fac75a0b0cdedacf9ab3e1ad9d929d149785903cff9bb69ad9727", size = 12868393, upload-time = "2025-09-04T16:49:23.043Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d2/9e3e40d399abc95336b1843f52fc0daaceb672d0e3c9290a28ff1a96f79d/ruff-0.12.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:abf4073688d7d6da16611f2f126be86523a8ec4343d15d276c614bda8ec44edb", size = 12036967, upload-time = "2025-09-04T16:49:26.04Z" }, + { url = "https://files.pythonhosted.org/packages/e9/03/6816b2ed08836be272e87107d905f0908be5b4a40c14bfc91043e76631b8/ruff-0.12.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:968e77094b1d7a576992ac078557d1439df678a34c6fe02fd979f973af167577", size = 12276038, upload-time = "2025-09-04T16:49:29.056Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d5/707b92a61310edf358a389477eabd8af68f375c0ef858194be97ca5b6069/ruff-0.12.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42a67d16e5b1ffc6d21c5f67851e0e769517fb57a8ebad1d0781b30888aa704e", size = 11901110, upload-time = "2025-09-04T16:49:32.07Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3d/f8b1038f4b9822e26ec3d5b49cf2bc313e3c1564cceb4c1a42820bf74853/ruff-0.12.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b216ec0a0674e4b1214dcc998a5088e54eaf39417327b19ffefba1c4a1e4971e", size = 13668352, upload-time = "2025-09-04T16:49:35.148Z" }, + { url = "https://files.pythonhosted.org/packages/98/0e/91421368ae6c4f3765dd41a150f760c5f725516028a6be30e58255e3c668/ruff-0.12.12-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:59f909c0fdd8f1dcdbfed0b9569b8bf428cf144bec87d9de298dcd4723f5bee8", size = 14638365, upload-time = "2025-09-04T16:49:38.892Z" }, + { url = "https://files.pythonhosted.org/packages/74/5d/88f3f06a142f58ecc8ecb0c2fe0b82343e2a2b04dcd098809f717cf74b6c/ruff-0.12.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ac93d87047e765336f0c18eacad51dad0c1c33c9df7484c40f98e1d773876f5", size = 14060812, upload-time = "2025-09-04T16:49:42.732Z" }, + { url = "https://files.pythonhosted.org/packages/13/fc/8962e7ddd2e81863d5c92400820f650b86f97ff919c59836fbc4c1a6d84c/ruff-0.12.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01543c137fd3650d322922e8b14cc133b8ea734617c4891c5a9fccf4bfc9aa92", size = 13050208, upload-time = "2025-09-04T16:49:46.434Z" }, + { url = "https://files.pythonhosted.org/packages/53/06/8deb52d48a9a624fd37390555d9589e719eac568c020b27e96eed671f25f/ruff-0.12.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afc2fa864197634e549d87fb1e7b6feb01df0a80fd510d6489e1ce8c0b1cc45", size = 13311444, upload-time = "2025-09-04T16:49:49.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/81/de5a29af7eb8f341f8140867ffb93f82e4fde7256dadee79016ac87c2716/ruff-0.12.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0c0945246f5ad776cb8925e36af2438e66188d2b57d9cf2eed2c382c58b371e5", size = 13279474, upload-time = "2025-09-04T16:49:53.465Z" }, + { url = "https://files.pythonhosted.org/packages/7f/14/d9577fdeaf791737ada1b4f5c6b59c21c3326f3f683229096cccd7674e0c/ruff-0.12.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a0fbafe8c58e37aae28b84a80ba1817f2ea552e9450156018a478bf1fa80f4e4", size = 12070204, upload-time = "2025-09-04T16:49:56.882Z" }, + { url = "https://files.pythonhosted.org/packages/77/04/a910078284b47fad54506dc0af13839c418ff704e341c176f64e1127e461/ruff-0.12.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b9c456fb2fc8e1282affa932c9e40f5ec31ec9cbb66751a316bd131273b57c23", size = 11880347, upload-time = "2025-09-04T16:49:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/df/58/30185fcb0e89f05e7ea82e5817b47798f7fa7179863f9d9ba6fd4fe1b098/ruff-0.12.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f12856123b0ad0147d90b3961f5c90e7427f9acd4b40050705499c98983f489", size = 12891844, upload-time = "2025-09-04T16:50:02.591Z" }, + { url = "https://files.pythonhosted.org/packages/21/9c/28a8dacce4855e6703dcb8cdf6c1705d0b23dd01d60150786cd55aa93b16/ruff-0.12.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:26a1b5a2bf7dd2c47e3b46d077cd9c0fc3b93e6c6cc9ed750bd312ae9dc302ee", size = 13360687, upload-time = "2025-09-04T16:50:05.8Z" }, + { url = "https://files.pythonhosted.org/packages/c8/fa/05b6428a008e60f79546c943e54068316f32ec8ab5c4f73e4563934fbdc7/ruff-0.12.12-py3-none-win32.whl", hash = "sha256:173be2bfc142af07a01e3a759aba6f7791aa47acf3604f610b1c36db888df7b1", size = 12052870, upload-time = "2025-09-04T16:50:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/85/60/d1e335417804df452589271818749d061b22772b87efda88354cf35cdb7a/ruff-0.12.12-py3-none-win_amd64.whl", hash = "sha256:e99620bf01884e5f38611934c09dd194eb665b0109104acae3ba6102b600fd0d", size = 13178016, upload-time = "2025-09-04T16:50:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762, upload-time = "2025-09-04T16:50:15.737Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "syrupy" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/f8/022d8704a3314f3e96dbd6bbd16ebe119ce30e35f41aabfa92345652fceb/syrupy-4.9.1.tar.gz", hash = "sha256:b7d0fcadad80a7d2f6c4c71917918e8ebe2483e8c703dfc8d49cdbb01081f9a4", size = 52492, upload-time = "2025-03-24T01:36:37.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/9d/aef9ec5fd5a4ee2f6a96032c4eda5888c5c7cec65cef6b28c4fc37671d88/syrupy-4.9.1-py3-none-any.whl", hash = "sha256:b94cc12ed0e5e75b448255430af642516842a2374a46936dd2650cfb6dd20eda", size = 52214, upload-time = "2025-03-24T01:36:35.278Z" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, +] + +[[package]] +name = "xmltodict" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/ee/b30fdb281b39da57053bd7012870989de6f066d6ef1476d78de8fc427324/xmltodict-0.15.0.tar.gz", hash = "sha256:c6d46b4e3413d1e4fc3e5016f0f1c7a5c10f8ce39efaa0cb099af986ecfc9a53", size = 60285, upload-time = "2025-09-05T00:35:45.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/56/507a207b96e3aa7365c28bb6702011e7c76c899c1737966b25852eaef3e8/xmltodict-0.15.0-py2.py3-none-any.whl", hash = "sha256:8887783bf1faba1754fc45fdf3fe03fbb3629c811ae57f91c018aace4c58d4ed", size = 10965, upload-time = "2025-09-05T00:35:44.583Z" }, +] From c3f3b2ebdc6ccce1fe896d221f7068cb4d4ccfbf Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 14 Sep 2025 21:18:31 -0400 Subject: [PATCH 49/69] lint, comment update --- .github/workflows/demos.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/demos.yml b/.github/workflows/demos.yml index 4c2703c..08d9880 100644 --- a/.github/workflows/demos.yml +++ b/.github/workflows/demos.yml @@ -30,4 +30,4 @@ jobs: secrets: inherit demo-jobs-split: uses: ./.github/workflows/demo-jobs-split.yml - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/README.md b/README.md index 7457e40..9d8bcf0 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ jobs: uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@main secrets: inherit with: - runner_grace_period: "120" # 2 minutes between jobs + runner_grace_period: "120" # Max idle time before termination (seconds) prepare: needs: ec2 From 15976112c29c878d2e612fccaaf55f70172fd9e2 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sun, 14 Sep 2025 21:18:56 -0400 Subject: [PATCH 50/69] rm vestigial `gpu-benchmark.py` --- .github/test-scripts/gpu-benchmark.py | 78 --------------------------- 1 file changed, 78 deletions(-) delete mode 100755 .github/test-scripts/gpu-benchmark.py diff --git a/.github/test-scripts/gpu-benchmark.py b/.github/test-scripts/gpu-benchmark.py deleted file mode 100755 index 3805291..0000000 --- a/.github/test-scripts/gpu-benchmark.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -"""GPU benchmark script for testing PyTorch CUDA capabilities. - -Used by ``demo-gpu.yml``. -""" - -import sys -import time -import torch - - -def main(): - print(f'PyTorch version: {torch.__version__}') - print(f'CUDA available: {torch.cuda.is_available()}') - - if not torch.cuda.is_available(): - print('ERROR: CUDA not available!') - sys.exit(1) - - # Get GPU information - device = torch.cuda.get_device_properties(0) - print(f'GPU: {device.name}') - print(f'Memory: {device.total_memory / 1e9:.1f} GB') - print(f'CUDA version: {torch.version.cuda}') - - # Matrix multiplication benchmark - print('\nRunning matrix multiplication benchmark...') - size = 8192 - x = torch.randn(size, size, dtype=torch.float32).cuda() - torch.cuda.synchronize() - - # Warmup - print(' Warming up...') - for _ in range(3): - _ = torch.matmul(x, x.T) - torch.cuda.synchronize() - - # Benchmark - print(' Running benchmark...') - start = time.time() - y = torch.matmul(x, x.T) - torch.cuda.synchronize() - elapsed = time.time() - start - - gflops = (2 * size**3) / (elapsed * 1e9) - print(f' Matrix size: {size}x{size}') - print(f' Time: {elapsed:.3f} seconds') - print(f' Performance: {gflops:.1f} GFLOPS') - print(f' Memory used: {torch.cuda.max_memory_allocated()/1e9:.2f} GB') - - # Quick training simulation - print('\nSimulating model training...') - model = torch.nn.Sequential( - torch.nn.Linear(1024, 512), - torch.nn.ReLU(), - torch.nn.Linear(512, 256), - torch.nn.ReLU(), - torch.nn.Linear(256, 10) - ).cuda() - - optimizer = torch.optim.Adam(model.parameters()) - data = torch.randn(32, 1024).cuda() - - for i in range(10): - output = model(data) - loss = output.sum() - loss.backward() - optimizer.step() - optimizer.zero_grad() - if i == 0 or i == 9: - print(f' Iteration {i+1}: loss = {loss.item():.4f}') - - print('\nGPU workload completed successfully!') - return 0 - - -if __name__ == '__main__': - sys.exit(main()) From 7401d0f03e38a772a29410c78687a3b3468f41dd Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 16 Sep 2025 21:25:12 -0400 Subject: [PATCH 51/69] `MAX_INSTANCE_LIFETIME = "120" # 2 hours (in minutes)` --- src/ec2_gha/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ec2_gha/defaults.py b/src/ec2_gha/defaults.py index 55233ee..fa98097 100644 --- a/src/ec2_gha/defaults.py +++ b/src/ec2_gha/defaults.py @@ -1,7 +1,7 @@ """Default values for ec2-gha configuration.""" # Instance lifetime and timing defaults -MAX_INSTANCE_LIFETIME = "360" # 6 hours (in minutes) +MAX_INSTANCE_LIFETIME = "120" # 2 hours (in minutes) RUNNER_GRACE_PERIOD = "60" # 1 minute (in seconds) RUNNER_INITIAL_GRACE_PERIOD = "180" # 3 minutes (in seconds) RUNNER_POLL_INTERVAL = "10" # 10 seconds From 0b464c641dbcab57d77798e878f617f0246006c9 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 16 Sep 2025 21:25:18 -0400 Subject: [PATCH 52/69] add trailing newlines --- src/ec2_gha/log_constants.py | 2 +- src/ec2_gha/templates/shared-functions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ec2_gha/log_constants.py b/src/ec2_gha/log_constants.py index 2e84451..c5de836 100644 --- a/src/ec2_gha/log_constants.py +++ b/src/ec2_gha/log_constants.py @@ -21,4 +21,4 @@ LOG_MSG_RUNNER_REMOVED = "Runner removed from GitHub successfully" # Default CloudWatch log group -DEFAULT_CLOUDWATCH_LOG_GROUP = "/aws/ec2/github-runners" \ No newline at end of file +DEFAULT_CLOUDWATCH_LOG_GROUP = "/aws/ec2/github-runners" diff --git a/src/ec2_gha/templates/shared-functions.sh b/src/ec2_gha/templates/shared-functions.sh index 7137467..50e0b72 100644 --- a/src/ec2_gha/templates/shared-functions.sh +++ b/src/ec2_gha/templates/shared-functions.sh @@ -174,4 +174,4 @@ EOF log "Started runner $idx in $runner_dir (PID: $pid)" return 0 -} \ No newline at end of file +} From 01dd7230f5f2b7bbadb97d8ffda25286204901fd Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 16 Sep 2025 21:25:42 -0400 Subject: [PATCH 53/69] `Check if any runners are actually running` --- src/ec2_gha/templates/user-script.sh.templ | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index c9889cf..4cf0839 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -239,6 +239,17 @@ source $$B/runner-common.sh A="$$V-last-activity" J="$$V-jobs" H="$$V-has-run-job" + +# Check if any runners are actually running +RUNNER_PROCS=\$$(pgrep -f "Runner.Listener" | wc -l) +if [ \$$RUNNER_PROCS -eq 0 ]; then + # No runner processes, check if we have stale job files + if ls \$$J/*.job 2>/dev/null | grep -q .; then + log "WARNING: Found job files but no runner processes - cleaning up stale jobs" + rm -f \$$J/*.job + fi +fi + [ ! -f "\$$A" ] && touch "\$$A" L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) N=\$$(date +%s) From 6059133e3dcb33f95ba658cd69cda2d0c6ce65de Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Tue, 16 Sep 2025 23:56:53 -0400 Subject: [PATCH 54/69] more robust `MAX_LIFETIME_MINUTES` shutdown logic some shared-functions factoring --- src/ec2_gha/templates/shared-functions.sh | 47 ++-- src/ec2_gha/templates/user-script.sh.templ | 20 +- tests/__snapshots__/test_start.ambr | 284 ++++++++++++++------- 3 files changed, 236 insertions(+), 115 deletions(-) diff --git a/src/ec2_gha/templates/shared-functions.sh b/src/ec2_gha/templates/shared-functions.sh index 50e0b72..7309686 100644 --- a/src/ec2_gha/templates/shared-functions.sh +++ b/src/ec2_gha/templates/shared-functions.sh @@ -6,17 +6,20 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } +dn=/dev/null + # Wait for dpkg lock to be released (for Debian/Ubuntu systems) wait_for_dpkg_lock() { - local timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" + local t=120 + local L=/var/lib/dpkg/lock + while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do + if [ $t -le 0 ]; then + log "WARNING: dpkg lock t, proceeding anyway" break fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($t seconds remaining)" sleep 5 - timeout=$((timeout - 5)) + t=$((t - 5)) done } @@ -24,18 +27,18 @@ wait_for_dpkg_lock() { flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true + systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true fi } # Get EC2 instance metadata (IMDSv2 compatible) get_metadata() { local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" fi return 0 # Always return success to avoid set -e issues } @@ -46,7 +49,7 @@ deregister_all_runners() { if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then log "Deregistering runner in $RUNNER_DIR" cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true sleep 1 if [ -f "$RUNNER_DIR/.runner-token" ]; then TOKEN=$(cat "$RUNNER_DIR/.runner-token") @@ -62,7 +65,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then log "Debug: Sleeping 600s before shutdown..." # Detect the SSH user from the home directory - local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" @@ -121,24 +124,24 @@ configure_runner() { # Install dependencies if needed if [ -f ./bin/installdependencies.sh ]; then # Quick check for common AMIs with pre-installed deps - if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then log "Dependencies exist, skipping install" else log "Installing dependencies..." set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 + sudo ./bin/installdependencies.sh >$dn 2>&1 local deps_result=$? set -e if [ $deps_result -ne 0 ]; then log "Dependencies script failed, installing manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then + if command -v dnf >$dn 2>&1; then + sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true + elif command -v yum >$dn 2>&1; then + sudo yum install -y libicu >$dn 2>&1 || true + elif command -v apt-get >$dn 2>&1; then wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + sudo apt-get update >$dn 2>&1 || true + sudo apt-get install -y libicu-dev >$dn 2>&1 || true fi fi fi @@ -169,7 +172,7 @@ EOF fi # Start runner in background - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & local pid=$! log "Started runner $idx in $runner_dir (PID: $pid)" diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 4cf0839..04c13cf 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -116,7 +116,25 @@ log "Instance metadata: Type=$${INSTANCE_TYPE} ID=$${INSTANCE_ID} Region=$${REGI # Set up maximum lifetime timeout - instance will terminate after this time regardless of job status MAX_LIFETIME_MINUTES=$max_instance_lifetime log "Setting up maximum lifetime timeout: $${MAX_LIFETIME_MINUTES} minutes" -nohup bash -c "sleep $${MAX_LIFETIME_MINUTES}m && echo '[$$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & +# Use ; instead of && so shutdown runs even if echo fails (e.g., disk full) +# Try multiple shutdown methods as fallbacks +nohup bash -c " + sleep $${MAX_LIFETIME_MINUTES}m + echo '[$$(date)] Maximum lifetime reached' 2>/dev/null || true + # Try normal shutdown + shutdown -h now 2>/dev/null || { + # If shutdown fails, try halt + halt -f 2>/dev/null || { + # If halt fails, try sysrq if available (Linux only) + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + # Last resort: force immediate reboot + reboot -f 2>/dev/null || true + } + } +" > /var/log/max-lifetime.log 2>&1 & # Configure CloudWatch Logs if a log group is specified if [ "$cloudwatch_logs_group" != "" ]; then diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index f806a00..50c7968 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -95,33 +95,36 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } + dn=/dev/null + wait_for_dpkg_lock() { - local timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" + local t=120 + local L=/var/lib/dpkg/lock + while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do + if [ $t -le 0 ]; then + log "WARNING: dpkg lock t, proceeding anyway" break fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($t seconds remaining)" sleep 5 - timeout=$((timeout - 5)) + t=$((t - 5)) done } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true + systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true fi } get_metadata() { local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" fi return 0 # Always return success to avoid set -e issues } @@ -131,7 +134,7 @@ if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then log "Deregistering runner in $RUNNER_DIR" cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true sleep 1 if [ -f "$RUNNER_DIR/.runner-token" ]; then TOKEN=$(cat "$RUNNER_DIR/.runner-token") @@ -145,7 +148,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" @@ -197,24 +200,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then log "Dependencies exist, skipping install" else log "Installing dependencies..." set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 + sudo ./bin/installdependencies.sh >$dn 2>&1 local deps_result=$? set -e if [ $deps_result -ne 0 ]; then log "Dependencies script failed, installing manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then + if command -v dnf >$dn 2>&1; then + sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true + elif command -v yum >$dn 2>&1; then + sudo yum install -y libicu >$dn 2>&1 || true + elif command -v apt-get >$dn 2>&1; then wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + sudo apt-get update >$dn 2>&1 || true + sudo apt-get install -y libicu-dev >$dn 2>&1 || true fi fi fi @@ -241,12 +244,13 @@ return 1 fi - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & local pid=$! log "Started runner $idx in $runner_dir (PID: $pid)" return 0 } + EOSF chmod +x $B/runner-common.sh @@ -293,7 +297,19 @@ MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & + nohup bash -c " + sleep ${MAX_LIFETIME_MINUTES}m + echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true + shutdown -h now 2>/dev/null || { + halt -f 2>/dev/null || { + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + reboot -f 2>/dev/null || true + } + } + " > /var/log/max-lifetime.log 2>&1 & if [ "" != "" ]; then log "Installing CloudWatch agent" @@ -401,6 +417,15 @@ A="$V-last-activity" J="$V-jobs" H="$V-has-run-job" + + RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) + if [ \$RUNNER_PROCS -eq 0 ]; then + if ls \$J/*.job 2>/dev/null | grep -q .; then + log "WARNING: Found job files but no runner processes - cleaning up stale jobs" + rm -f \$J/*.job + fi + fi + [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) N=\$(date +%s) @@ -625,33 +650,36 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } + dn=/dev/null + wait_for_dpkg_lock() { - local timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" + local t=120 + local L=/var/lib/dpkg/lock + while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do + if [ $t -le 0 ]; then + log "WARNING: dpkg lock t, proceeding anyway" break fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($t seconds remaining)" sleep 5 - timeout=$((timeout - 5)) + t=$((t - 5)) done } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true + systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true fi } get_metadata() { local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" fi return 0 # Always return success to avoid set -e issues } @@ -661,7 +689,7 @@ if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then log "Deregistering runner in $RUNNER_DIR" cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true sleep 1 if [ -f "$RUNNER_DIR/.runner-token" ]; then TOKEN=$(cat "$RUNNER_DIR/.runner-token") @@ -675,7 +703,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" @@ -727,24 +755,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then log "Dependencies exist, skipping install" else log "Installing dependencies..." set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 + sudo ./bin/installdependencies.sh >$dn 2>&1 local deps_result=$? set -e if [ $deps_result -ne 0 ]; then log "Dependencies script failed, installing manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then + if command -v dnf >$dn 2>&1; then + sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true + elif command -v yum >$dn 2>&1; then + sudo yum install -y libicu >$dn 2>&1 || true + elif command -v apt-get >$dn 2>&1; then wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + sudo apt-get update >$dn 2>&1 || true + sudo apt-get install -y libicu-dev >$dn 2>&1 || true fi fi fi @@ -771,12 +799,13 @@ return 1 fi - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & local pid=$! log "Started runner $idx in $runner_dir (PID: $pid)" return 0 } + EOSF chmod +x $B/runner-common.sh @@ -823,7 +852,19 @@ MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & + nohup bash -c " + sleep ${MAX_LIFETIME_MINUTES}m + echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true + shutdown -h now 2>/dev/null || { + halt -f 2>/dev/null || { + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + reboot -f 2>/dev/null || true + } + } + " > /var/log/max-lifetime.log 2>&1 & if [ "" != "" ]; then log "Installing CloudWatch agent" @@ -931,6 +972,15 @@ A="$V-last-activity" J="$V-jobs" H="$V-has-run-job" + + RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) + if [ \$RUNNER_PROCS -eq 0 ]; then + if ls \$J/*.job 2>/dev/null | grep -q .; then + log "WARNING: Found job files but no runner processes - cleaning up stale jobs" + rm -f \$J/*.job + fi + fi + [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) N=\$(date +%s) @@ -1119,33 +1169,36 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } + dn=/dev/null + wait_for_dpkg_lock() { - local timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" + local t=120 + local L=/var/lib/dpkg/lock + while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do + if [ $t -le 0 ]; then + log "WARNING: dpkg lock t, proceeding anyway" break fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($t seconds remaining)" sleep 5 - timeout=$((timeout - 5)) + t=$((t - 5)) done } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true + systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true fi } get_metadata() { local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" fi return 0 # Always return success to avoid set -e issues } @@ -1155,7 +1208,7 @@ if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then log "Deregistering runner in $RUNNER_DIR" cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true sleep 1 if [ -f "$RUNNER_DIR/.runner-token" ]; then TOKEN=$(cat "$RUNNER_DIR/.runner-token") @@ -1169,7 +1222,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" @@ -1221,24 +1274,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then log "Dependencies exist, skipping install" else log "Installing dependencies..." set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 + sudo ./bin/installdependencies.sh >$dn 2>&1 local deps_result=$? set -e if [ $deps_result -ne 0 ]; then log "Dependencies script failed, installing manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then + if command -v dnf >$dn 2>&1; then + sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true + elif command -v yum >$dn 2>&1; then + sudo yum install -y libicu >$dn 2>&1 || true + elif command -v apt-get >$dn 2>&1; then wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + sudo apt-get update >$dn 2>&1 || true + sudo apt-get install -y libicu-dev >$dn 2>&1 || true fi fi fi @@ -1265,12 +1318,13 @@ return 1 fi - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & local pid=$! log "Started runner $idx in $runner_dir (PID: $pid)" return 0 } + EOSF chmod +x $B/runner-common.sh @@ -1317,7 +1371,19 @@ MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & + nohup bash -c " + sleep ${MAX_LIFETIME_MINUTES}m + echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true + shutdown -h now 2>/dev/null || { + halt -f 2>/dev/null || { + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + reboot -f 2>/dev/null || true + } + } + " > /var/log/max-lifetime.log 2>&1 & if [ "" != "" ]; then log "Installing CloudWatch agent" @@ -1425,6 +1491,15 @@ A="$V-last-activity" J="$V-jobs" H="$V-has-run-job" + + RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) + if [ \$RUNNER_PROCS -eq 0 ]; then + if ls \$J/*.job 2>/dev/null | grep -q .; then + log "WARNING: Found job files but no runner processes - cleaning up stale jobs" + rm -f \$J/*.job + fi + fi + [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) N=\$(date +%s) @@ -1612,33 +1687,36 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } log_error() { log "ERROR: $1" >&2; } + dn=/dev/null + wait_for_dpkg_lock() { - local timeout=120 - while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - if [ $timeout -le 0 ]; then - log "WARNING: dpkg lock timeout, proceeding anyway" + local t=120 + local L=/var/lib/dpkg/lock + while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do + if [ $t -le 0 ]; then + log "WARNING: dpkg lock t, proceeding anyway" break fi - log "dpkg is locked, waiting... ($timeout seconds remaining)" + log "dpkg is locked, waiting... ($t seconds remaining)" sleep 5 - timeout=$((timeout - 5)) + t=$((t - 5)) done } flush_cloudwatch_logs() { log "Stopping CloudWatch agent to flush logs" if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>/dev/null || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>/dev/null || true + systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true fi } get_metadata() { local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>/dev/null || true) + local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>/dev/null || echo "unknown" + curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" fi return 0 # Always return success to avoid set -e issues } @@ -1648,7 +1726,7 @@ if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then log "Deregistering runner in $RUNNER_DIR" cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>/dev/null || true + pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true sleep 1 if [ -f "$RUNNER_DIR/.runner-token" ]; then TOKEN=$(cat "$RUNNER_DIR/.runner-token") @@ -1662,7 +1740,7 @@ debug_sleep_and_shutdown() { if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>/dev/null || echo "ec2-user") + local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" @@ -1714,24 +1792,24 @@ tar -xzf /tmp/runner.tar.gz if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >/dev/null 2>&1 && dpkg -l libicu[0-9]* 2>/dev/null | grep -q ^ii; then + if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then log "Dependencies exist, skipping install" else log "Installing dependencies..." set +e - sudo ./bin/installdependencies.sh >/dev/null 2>&1 + sudo ./bin/installdependencies.sh >$dn 2>&1 local deps_result=$? set -e if [ $deps_result -ne 0 ]; then log "Dependencies script failed, installing manually..." - if command -v dnf >/dev/null 2>&1; then - sudo dnf install -y libicu lttng-ust >/dev/null 2>&1 || true - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libicu >/dev/null 2>&1 || true - elif command -v apt-get >/dev/null 2>&1; then + if command -v dnf >$dn 2>&1; then + sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true + elif command -v yum >$dn 2>&1; then + sudo yum install -y libicu >$dn 2>&1 || true + elif command -v apt-get >$dn 2>&1; then wait_for_dpkg_lock - sudo apt-get update >/dev/null 2>&1 || true - sudo apt-get install -y libicu-dev >/dev/null 2>&1 || true + sudo apt-get update >$dn 2>&1 || true + sudo apt-get install -y libicu-dev >$dn 2>&1 || true fi fi fi @@ -1758,12 +1836,13 @@ return 1 fi - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > /dev/null 2>&1 & + RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & local pid=$! log "Started runner $idx in $runner_dir (PID: $pid)" return 0 } + EOSF chmod +x $B/runner-common.sh @@ -1810,7 +1889,19 @@ MAX_LIFETIME_MINUTES=360 log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c "sleep ${MAX_LIFETIME_MINUTES}m && echo '[$(date)] Maximum lifetime reached' && shutdown -h now" > /var/log/max-lifetime.log 2>&1 & + nohup bash -c " + sleep ${MAX_LIFETIME_MINUTES}m + echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true + shutdown -h now 2>/dev/null || { + halt -f 2>/dev/null || { + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + reboot -f 2>/dev/null || true + } + } + " > /var/log/max-lifetime.log 2>&1 & if [ "/aws/ec2/github-runners" != "" ]; then log "Installing CloudWatch agent" @@ -1918,6 +2009,15 @@ A="$V-last-activity" J="$V-jobs" H="$V-has-run-job" + + RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) + if [ \$RUNNER_PROCS -eq 0 ]; then + if ls \$J/*.job 2>/dev/null | grep -q .; then + log "WARNING: Found job files but no runner processes - cleaning up stale jobs" + rm -f \$J/*.job + fi + fi + [ ! -f "\$A" ] && touch "\$A" L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) N=\$(date +%s) From 854396e454ca752871846c453d5c894f4f0ee3cc Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 11:45:13 -0400 Subject: [PATCH 55/69] Add placeholder workflow for demo-disk-full --- .github/workflows/test-disk-full.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/workflows/test-disk-full.yml diff --git a/.github/workflows/test-disk-full.yml b/.github/workflows/test-disk-full.yml new file mode 100644 index 0000000..1c0b0ab --- /dev/null +++ b/.github/workflows/test-disk-full.yml @@ -0,0 +1,8 @@ +name: Test Disk Full +on: + workflow_dispatch: +jobs: + placeholder: + runs-on: ubuntu-latest + steps: + - run: echo "Placeholder workflow to preserve workflow name in GitHub Actions UI" From fba0891c01b5783eabc51f42162e262d8af70b80 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 16:31:02 -0400 Subject: [PATCH 56/69] tweak cpu-sweep Debian job names --- .github/workflows/demo-cpu-sweep.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/demo-cpu-sweep.yml b/.github/workflows/demo-cpu-sweep.yml index 2f4e243..939748b 100644 --- a/.github/workflows/demo-cpu-sweep.yml +++ b/.github/workflows/demo-cpu-sweep.yml @@ -27,10 +27,10 @@ jobs: - { "os": Ubuntu 22.04, "arch": ARM, "ami": ami-06daf9c2d2cf1cb37, "instance_type": t4g.medium } - { "os": Ubuntu 24.04, "arch": x86, "ami": ami-0ca5a2f40c2601df6, "instance_type": t3.medium } - { "os": Ubuntu 24.04, "arch": ARM, "ami": ami-0aa307ed50ca3e58f, "instance_type": t4g.medium } - - { "os": Debian 11, "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } - - { "os": Debian 11, "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } - - { "os": Debian 12, "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } - - { "os": Debian 12, "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } + - { "os": Debian 11 , "arch": x86, "ami": ami-0e6612f57082e7ea4, "instance_type": t3.large } + - { "os": Debian 11 , "arch": ARM, "ami": ami-0c3f5b0b87f042da8, "instance_type": t4g.large } + - { "os": Debian 12 , "arch": x86, "ami": ami-05b50089e01b13194, "instance_type": t3.large } + - { "os": Debian 12 , "arch": ARM, "ami": ami-0505441d7e1514742, "instance_type": t4g.large } - { "os": AL2 , "arch": x86, "ami": ami-0e2c86481225d3c51, "instance_type": t3.small } - { "os": AL2 , "arch": ARM, "ami": ami-08333c9352b93f31e, "instance_type": t4g.small } - { "os": AL2023 , "arch": x86, "ami": ami-00ca32bbc84273381, "instance_type": t3.small } From 56c5ca7077e02cd2fbd2f271231874be1571c4e7 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 16:31:24 -0400 Subject: [PATCH 57/69] `CLAUDE.md`: clearer `update-snapshots` comment --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 696a6db..0e29f5f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ cd tests/ && pytest -v -m 'not slow' # pytest --snapshot-update -m 'not slow' # pytest -vvv -m 'not slow' . # ``` -# Can be used in conjunction with `git rebase -x`. I'll mostly do this manually when cleaning up commits. +# Update syrupy "snapshot" files. Can also be used in conjunction with `git rebase -x` (I'll mostly do that manually, when cleaning up commits). scripts/update-snapshots.sh ``` From 7a6d11ea745fc51af130867d2f0ead5a16d3f130 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 17:18:38 -0400 Subject: [PATCH 58/69] DL userdata payload at runtime --- .github/workflows/runner.yml | 1 + CLAUDE.md | 43 +- README.md | 28 +- action.yml | 4 + .../scripts/check-runner-termination.sh | 49 + src/ec2_gha/scripts/job-completed-hook.sh | 19 + src/ec2_gha/scripts/job-started-hook.sh | 21 + src/ec2_gha/scripts/runner-setup.sh | 383 +++ src/ec2_gha/start.py | 89 +- src/ec2_gha/templates/user-script.sh.templ | 453 +--- tests/__snapshots__/test_start.ambr | 2228 ++--------------- tests/test_start.py | 71 +- 12 files changed, 919 insertions(+), 2470 deletions(-) create mode 100644 src/ec2_gha/scripts/check-runner-termination.sh create mode 100644 src/ec2_gha/scripts/job-completed-hook.sh create mode 100644 src/ec2_gha/scripts/job-started-hook.sh create mode 100755 src/ec2_gha/scripts/runner-setup.sh diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index cfec631..c7d45c1 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -166,6 +166,7 @@ jobs: id: aws-start uses: ./ with: + action_ref: ${{ inputs.action_ref }} aws_region: ${{ inputs.aws_region || vars.AWS_REGION }} aws_tags: ${{ inputs.aws_tags }} cloudwatch_logs_group: ${{ inputs.cloudwatch_logs_group || vars.CLOUDWATCH_LOGS_GROUP }} diff --git a/CLAUDE.md b/CLAUDE.md index 0e29f5f..566adf6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,14 +51,25 @@ ruff format src/ - **`src/ec2_gha/__main__.py`**: Entry point that parses environment variables and initiates runner creation - **`src/ec2_gha/start.py`**: Contains `StartAWS` class handling EC2 operations, instance lifecycle, and template rendering -### Template System -- **`src/ec2_gha/templates/user-script.sh.templ`**: Main userdata template using Python's String.Template format, includes inline runner lifecycle hooks - -## Versioning - -`runner.yml` runs `actions/checkout` on this repo but, when called from another workflow, GitHub Actions provides no way to checkout the same ref that this workflow was called at. To avoid this, `inputs.action_ref` gets a default value corresponding to a branch or tag name that each commit (in this repo) is expected to be referenced as (e.g. `v2`). This allows the workflow to check out the correct code without the user needing to explicitly pass an `action_ref` input. - -Patch/minor version tags like `v2.0.0`, `v2.1.0` can be created from the `v2` branch, with the `action_ref` default pinned to those values (similar to syncing a project's `pyproject.toml` version with a Git tag pointing at a given commit). +### Template and Script System +- **`src/ec2_gha/templates/user-script.sh.templ`**: Main userdata template using Python's String.Template format +- **`src/ec2_gha/scripts/runner-setup.sh`**: Main runner setup script fetched by userdata +- **`src/ec2_gha/scripts/job-started-hook.sh`**: GitHub Actions hook for job start events +- **`src/ec2_gha/scripts/job-completed-hook.sh`**: GitHub Actions hook for job completion +- **`src/ec2_gha/scripts/check-runner-termination.sh`**: Periodic termination check script + +## Versioning and Security + +### Action Ref Resolution +`runner.yml` requires an `action_ref` parameter that gets resolved to a Git SHA for security: +1. Python code resolves branch/tag references to immutable SHAs +2. All scripts are fetched using the resolved SHA to prevent TOCTOU attacks +3. This ensures the exact code version is used throughout execution + +### Version Strategy +- Main branch (`v2`) contains stable releases +- `action_ref` defaults to the branch name in `runner.yml` +- Patch/minor version tags like `v2.0.0`, `v2.1.0` can be created from the `v2` branch `ec2-gha`'s initial release uses a `v2` branch because the upstream `start-aws-gha-runner` has published some `v1*` tags. @@ -66,7 +77,8 @@ Patch/minor version tags like `v2.0.0`, `v2.1.0` can be created from the `v2` br ```yaml # Caller workflow uses the v2 branch uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@v2 -# The runner.yml on v2 branch has action_ref default of "v2", so it automatically checks out the correct code +# The runner.yml on v2 branch has action_ref default of "v2" +# This gets resolved to a SHA at runtime for security ``` For complete usage examples, see `.github/workflows/demo*.yml`. @@ -97,9 +109,9 @@ GitHub Actions declares env vars prefixed with `INPUT_` for each input, which `s #### Termination Logic The runner uses a polling-based approach to determine when to terminate: -1. **Job Tracking**: GitHub runner hooks (`job-started`, `job-completed`) track job lifecycle - - Creates/updates JSON files in `/var/run/github-runner-jobs/` - - Updates "last activity" timestamp at `/var/run/github-runner-last-activity` +1. **Job Tracking**: GitHub runner hooks track job lifecycle + - `job-started-hook.sh`: Creates JSON files in `/var/run/github-runner-jobs/` + - `job-completed-hook.sh`: Removes job files and updates activity timestamp 2. **Periodic Polling**: Systemd timer runs `check-runner-termination.sh` every `runner_poll_interval` seconds (default: 10s) - Checks for running jobs (files with `"status":"running"`) @@ -114,10 +126,9 @@ The runner uses a polling-based approach to determine when to terminate: - Example: With 60s grace and 10s poll → terminates 60-70s after last job 4. **Clean Shutdown Sequence**: - - Stop runner process gracefully (SIGINT) - - Remove runner from GitHub (`config.sh remove`) - - Flush CloudWatch logs - - Execute `shutdown -h now` + - Stop runner processes gracefully (SIGINT with timeout) + - Deregister all runners from GitHub + - Flush CloudWatch logs (if configured) ### AWS Resource Tagging By default, launched EC2 instances are Tagged with: diff --git a/README.md b/README.md index 9d8bcf0..ed5b193 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ The `EC2_LAUNCH_ROLE` is passed to [aws-actions/configure-aws-credentials]; if y Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): -- `action_ref` - ec2-gha Git ref to checkout (branch/tag/SHA); auto-detected if not specified +- `action_ref` - ec2-gha Git ref to checkout (branch/tag/SHA); automatically resolved to a SHA for security - `aws_region` - AWS region for EC2 instances (falls back to `vars.AWS_REGION`, default: `us-east-1`) - `cloudwatch_logs_group` - CloudWatch Logs group name for streaming logs (falls back to `vars.CLOUDWATCH_LOGS_GROUP`) - `ec2_home_dir` - Home directory (default: `/home/ubuntu`) @@ -204,20 +204,24 @@ jobs: ### Termination logic -The runner uses [GitHub Actions runner hooks][hooks] to track job start/end events, and a `systemd` timer to poll for when there's: -1. no active jobs running, and -2. no job starts or ends in at least `runner_grace_period` seconds. +The runner uses [GitHub Actions runner hooks][hooks] to track job lifecycle and determine when to terminate: -Job start/end events `touch` a "last activity" timestamp file (`/var/run/github-runner-last-activity`), and the systemd timer checks this file every `runner_poll_interval` seconds (default: 10s). +#### Job Tracking +- **Start/End Hooks**: Creates/removes JSON files in `/var/run/github-runner-jobs/` when jobs start/end +- **Activity Tracking**: Updates `/var/run/github-runner-last-activity` timestamp on job events -Each job's status is tracked in a JSON file like `/var/run/github-runner-jobs/.job`. +#### Termination Conditions +The systemd timer checks every `runner_poll_interval` seconds (default: 10s) and terminates when: +1. No active jobs are running +2. Idle time exceeds the grace period: + - `runner_initial_grace_period` (default: 180s) - Before first job + - `runner_grace_period` (default: 60s) - Between jobs -The default `runner_grace_period` is 60s, but a longer `runner_initial_grace_period` (default: 180s) is used for the first job after instance boot (to allow time for the runner to register and start). - -When terminating, the runner: -- Gracefully stops the runner process -- Removes itself from GitHub -- Flushes CloudWatch logs +#### Clean Shutdown Sequence +1. Stop runner processes gracefully (SIGINT) +2. Deregister runners from GitHub +3. Flush CloudWatch logs (if configured) +4. Execute shutdown with multiple fallback methods ### CloudWatch Logs Integration diff --git a/action.yml b/action.yml index e6caa5b..438bfc1 100644 --- a/action.yml +++ b/action.yml @@ -4,6 +4,10 @@ runs: using: "docker" image: "Dockerfile" inputs: + action_ref: + description: "ec2-gha Git ref (branch/tag/SHA) to use for fetching scripts" + required: false + default: "v2" aws_region: description: "AWS region for EC2 instances (falls back to vars.AWS_REGION, then us-east-1)" required: false diff --git a/src/ec2_gha/scripts/check-runner-termination.sh b/src/ec2_gha/scripts/check-runner-termination.sh new file mode 100644 index 0000000..36ddf55 --- /dev/null +++ b/src/ec2_gha/scripts/check-runner-termination.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Periodic check for GitHub Actions runner termination conditions +# Called by systemd timer to determine if the instance should shut down + +exec >> /tmp/termination-check.log 2>&1 + +# Source common functions and variables +source /usr/local/bin/runner-common.sh + +# File paths for tracking +A="/var/run/github-runner-last-activity" +J="/var/run/github-runner-jobs" +H="/var/run/github-runner-has-run-job" + +# Current timestamp +N=$(date +%s) + +# Check if any runners are actually running +RUNNER_PROCS=$(pgrep -f "Runner.Listener" | wc -l) +if [ $RUNNER_PROCS -eq 0 ]; then + # No runner processes, check if we have stale job files + if ls $J/*.job 2>/dev/null | grep -q .; then + log "WARNING: Found job files but no runner processes - cleaning up stale jobs" + rm -f $J/*.job + fi +fi + +# Ensure activity file exists and get its timestamp +[ ! -f "$A" ] && touch "$A" +L=$(stat -c %Y "$A" 2>/dev/null || echo 0) + +# Calculate idle time +I=$((N-L)) + +# Determine grace period based on whether any job has run yet +[ -f "$H" ] && G=${RUNNER_GRACE_PERIOD:-60} || G=${RUNNER_INITIAL_GRACE_PERIOD:-180} + +# Count running jobs +R=$(grep -l '"status":"running"' $J/*.job 2>/dev/null | wc -l || echo 0) + +# Check if we should terminate +if [ $R -eq 0 ] && [ $I -gt $G ]; then + log "TERMINATING: idle $I > grace $G" + deregister_all_runners + flush_cloudwatch_logs + debug_sleep_and_shutdown +else + [ $R -gt 0 ] && log "$R job(s) running" || log "Idle $I/$G sec" +fi diff --git a/src/ec2_gha/scripts/job-completed-hook.sh b/src/ec2_gha/scripts/job-completed-hook.sh new file mode 100644 index 0000000..f0a83f6 --- /dev/null +++ b/src/ec2_gha/scripts/job-completed-hook.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# GitHub Actions runner job-completed hook +# Called when a job finishes (success or failure) on this runner +# Environment variables provided by GitHub Actions runner + +exec >> /tmp/job-completed-hook.log 2>&1 + +# Get runner index from environment (defaults to 0 for single-runner instances) +I="${RUNNER_INDEX:-0}" + +# Log the job completion with a specific prefix for CloudWatch filtering +# The LOG_PREFIX will be substituted during setup +echo "[$(date)] Runner-$I: LOG_PREFIX_JOB_COMPLETED ${GITHUB_JOB}" + +# Remove the job tracking file to indicate this runner no longer has an active job +rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$I.job + +# Update activity timestamp to reset the idle timer +touch /var/run/github-runner-last-activity diff --git a/src/ec2_gha/scripts/job-started-hook.sh b/src/ec2_gha/scripts/job-started-hook.sh new file mode 100644 index 0000000..55315d6 --- /dev/null +++ b/src/ec2_gha/scripts/job-started-hook.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# GitHub Actions runner job-started hook +# Called when a job starts running on this runner +# Environment variables provided by GitHub Actions runner + +exec >> /tmp/job-started-hook.log 2>&1 + +# Get runner index from environment (defaults to 0 for single-runner instances) +I="${RUNNER_INDEX:-0}" + +# Log the job start with a specific prefix for CloudWatch filtering +# The LOG_PREFIX will be substituted during setup +echo "[$(date)] Runner-$I: LOG_PREFIX_JOB_STARTED Runner-$I: ${GITHUB_JOB}" + +# Create a job tracking file to indicate this runner has an active job +# Format: RUNID-JOBNAME-RUNNER.job +mkdir -p /var/run/github-runner-jobs +echo '{"status":"running","runner":"'$I'"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$I.job + +# Update activity timestamps to reset the idle timer +touch /var/run/github-runner-last-activity /var/run/github-runner-has-run-job diff --git a/src/ec2_gha/scripts/runner-setup.sh b/src/ec2_gha/scripts/runner-setup.sh new file mode 100755 index 0000000..35427c0 --- /dev/null +++ b/src/ec2_gha/scripts/runner-setup.sh @@ -0,0 +1,383 @@ +#!/bin/bash +set -e + +# This script is fetched and executed by the minimal userdata script +# All variables are already exported by the userdata script + +# Enable debug tracing to a file for troubleshooting +exec 2> >(tee -a /var/log/runner-debug.log >&2) + +# Conditionally enable debug mode +[ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ] && set -x + +# Determine home directory early since it's needed by shared functions +if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + # Try to find the default non-root user's home directory + for user in ubuntu ec2-user centos admin debian fedora alpine arch; do + if id "$user" &>/dev/null; then + homedir="/home/$user" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log + break + fi + done + + # Fallback if no standard user found + if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then + homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\/home\// {print $6}' | while read dir; do + if [ -d "$dir" ]; then + echo "$dir" + break + fi + done) + if [ -z "$homedir" ]; then + homedir="/home/ec2-user" # Ultimate fallback + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log + else + owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log + fi + fi +else + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log +fi +export homedir + +# Set common paths +B=/usr/local/bin +V=/var/run/github-runner + +# Fetch shared functions from GitHub +echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching shared functions from GitHub (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log +FUNCTIONS_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/${action_sha}/src/ec2_gha/templates/shared-functions.sh" +if ! curl -sSL "$FUNCTIONS_URL" -o /tmp/shared-functions.sh && ! wget -q "$FUNCTIONS_URL" -O /tmp/shared-functions.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download shared functions" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 +fi + +# Write shared functions that will be used by multiple scripts +cat > $B/runner-common.sh << EOSF +# Auto-generated shared functions and variables +# Set homedir for scripts that source this file +homedir="$homedir" +debug="$debug" +export homedir debug + +EOSF + +# Append the downloaded shared functions +cat /tmp/shared-functions.sh >> $B/runner-common.sh + +chmod +x $B/runner-common.sh +source $B/runner-common.sh + +logger "EC2-GHA: Starting userdata script" +trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR +trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR +# Handle watchdog termination signal +trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM + +# Set up registration timeout failsafe - terminate if runner doesn't register in time +REGISTRATION_TIMEOUT="$runner_registration_timeout" +if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then + REGISTRATION_TIMEOUT=300 +fi +# Create a marker file for watchdog termination request +touch $V-watchdog-active +( + sleep $REGISTRATION_TIMEOUT + if [ ! -f $V-registered ]; then + touch $V-watchdog-terminate + kill -TERM $$ 2>/dev/null || true + fi + rm -f $V-watchdog-active +) & +REGISTRATION_WATCHDOG_PID=$! +echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid + +# Run any custom user data script provided by the user +if [ -n "$userdata" ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Running custom userdata" | tee -a /var/log/runner-setup.log + eval "$userdata" +fi + +exec >> /var/log/runner-setup.log 2>&1 +log "Starting runner setup" + +# Fetch instance metadata for labeling and logging +INSTANCE_TYPE=$(get_metadata "instance-type") +INSTANCE_ID=$(get_metadata "instance-id") +REGION=$(get_metadata "placement/region") +AZ=$(get_metadata "placement/availability-zone") +log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" + +# Set up maximum lifetime timeout - instance will terminate after this time regardless of job status +MAX_LIFETIME_MINUTES=$max_instance_lifetime +log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" +# Use ; instead of && so shutdown runs even if echo fails (e.g., disk full) +# Try multiple shutdown methods as fallbacks +nohup bash -c " + sleep ${MAX_LIFETIME_MINUTES}m + echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true + # Try normal shutdown + shutdown -h now 2>/dev/null || { + # If shutdown fails, try halt + halt -f 2>/dev/null || { + # If halt fails, try sysrq if available (Linux only) + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + # Last resort: force immediate reboot + reboot -f 2>/dev/null || true + } + } +" > /var/log/max-lifetime.log 2>&1 & + +# Configure CloudWatch Logs if a log group is specified +if [ "$cloudwatch_logs_group" != "" ]; then + log "Installing CloudWatch agent" + ( + if command -v dpkg >/dev/null 2>&1; then + wait_for_dpkg_lock + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + fi + # Build CloudWatch config with factored strings + P='"file_path":' + G=',"log_group_name":"'$cloudwatch_logs_group'","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$homedir" + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF +{"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ +{"$P/var/log/runner-setup.log${G}runner-setup$Z", +{"$P/var/log/runner-debug.log${G}runner-debug$Z", +{"$P/tmp/job-started-hook.log${G}job-started$Z", +{"$P/tmp/job-completed-hook.log${G}job-completed$Z", +{"$P/tmp/termination-check.log${G}termination$Z", +{"$P/tmp/runner-*-config.log${G}runner-config$Z", +{"$P$H/_diag/Runner_**.log${G}runner-diag$Z", +{"$P$H/_diag/Worker_**.log${G}worker-diag$Z" +]}}}} +EOF + /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s + log "CloudWatch agent started" + ) || log "WARNING: CloudWatch agent installation failed, continuing without it" +fi + +# Configure SSH access if public key provided (useful for debugging) +if [ -n "$ssh_pubkey" ]; then + log "Configuring SSH access" + # Determine the default user based on the home directory owner + DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") + mkdir -p "$homedir/.ssh" + chmod 700 "$homedir/.ssh" + echo "$ssh_pubkey" >> "$homedir/.ssh/authorized_keys" + chmod 600 "$homedir/.ssh/authorized_keys" + if [ "$DEFAULT_USER" != "root" ]; then + chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" + fi + log "SSH key added for user $DEFAULT_USER" +fi + +log "Working directory: $homedir" +cd "$homedir" + +# Run any pre-runner script provided by the user +if [ -n "$script" ]; then + echo "$script" > pre-runner-script.sh + log "Running pre-runner script" + source pre-runner-script.sh +fi +export RUNNER_ALLOW_RUNASROOT=1 + +# Number of runners to configure on this instance +RUNNERS_PER_INSTANCE=$runners_per_instance + +# Download GitHub Actions runner binary +ARCH=$(uname -m) +if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + RUNNER_URL=$(echo "$runner_release" | sed 's/x64/arm64/g') + log "ARM detected, using: $RUNNER_URL" +else + RUNNER_URL="$runner_release" + log "x64 detected, using: $RUNNER_URL" +fi + +if command -v curl >/dev/null 2>&1; then + curl -L $RUNNER_URL -o /tmp/runner.tar.gz +elif command -v wget >/dev/null 2>&1; then + wget -q $RUNNER_URL -O /tmp/runner.tar.gz +else + log_error "Neither curl nor wget found. Cannot download runner." + terminate_instance "No download tool available" +fi +log "Downloaded runner binary" + +# Helper function to fetch scripts +fetch_script() { + local script_name="$1" + local url="${BASE_URL}/${script_name}" + local dest="${B}/${script_name}" + + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$url" -o "$dest" || { + log_error "Failed to fetch $script_name" + terminate_instance "Failed to download $script_name" + } + elif command -v wget >/dev/null 2>&1; then + wget -q "$url" -O "$dest" || { + log_error "Failed to fetch $script_name" + terminate_instance "Failed to download $script_name" + } + else + log_error "Neither curl nor wget found. Cannot download scripts." + terminate_instance "No download tool available" + fi +} + +# Fetch job tracking scripts from GitHub +# These scripts are called by GitHub runner hooks +log "Fetching runner hook scripts" +BASE_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/${action_sha}/src/ec2_gha/scripts" + +fetch_script "job-started-hook.sh" +fetch_script "job-completed-hook.sh" +fetch_script "check-runner-termination.sh" + +# Replace log prefix placeholders with actual values +sed -i "s/LOG_PREFIX_JOB_STARTED/${log_prefix_job_started}/g" $B/job-started-hook.sh +sed -i "s/LOG_PREFIX_JOB_COMPLETED/${log_prefix_job_completed}/g" $B/job-completed-hook.sh + +chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh + +# Set up job tracking directory +mkdir -p $V-jobs +touch $V-last-activity + +# Set up periodic termination check using systemd +cat > /etc/systemd/system/runner-termination-check.service << EOF +[Unit] +Description=Check GitHub runner termination conditions +After=network.target +[Service] +Type=oneshot +Environment="RUNNER_GRACE_PERIOD=$runner_grace_period" +Environment="RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" +Environment="RUNNER_POLL_INTERVAL=$runner_poll_interval" +ExecStart=$B/check-runner-termination.sh +EOF + +cat > /etc/systemd/system/runner-termination-check.timer << EOF +[Unit] +Description=Periodic GitHub runner termination check +Requires=runner-termination-check.service +[Timer] +OnBootSec=60s +OnUnitActiveSec=${runner_poll_interval}s +[Install] +WantedBy=timers.target +EOF + +systemctl daemon-reload +systemctl enable runner-termination-check.timer +systemctl start runner-termination-check.timer + +# Build metadata labels (these will be added to the runner labels) +METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" +# Add instance name as a label if provided +if [ -n "$instance_name" ]; then + INSTANCE_NAME_LABEL=$(echo "$instance_name" | tr ' /' '-' | tr -cd '[:alnum:]-_#') + METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" +fi + +log "Setting up $RUNNERS_PER_INSTANCE runner(s)" + +# Export functions for subprocesses (variables already exported from runner-common.sh) +export -f configure_runner +export -f log +export -f log_error +export -f get_metadata +export -f flush_cloudwatch_logs +export -f deregister_all_runners +export -f debug_sleep_and_shutdown +export -f wait_for_dpkg_lock + +# Parse space-delimited tokens and pipe-delimited labels +IFS=' ' read -ra tokens <<< "$runner_tokens" +IFS='|' read -ra labels <<< "$runner_labels" + +num_runners=${#tokens[@]} +log "Configuring $num_runners runner(s) in parallel" + +# Start configuration for each runner in parallel +pids=() +for i in ${!tokens[@]}; do + token=${tokens[$i]} + label=${labels[$i]:-} + if [ -z "$token" ]; then + log_error "No token for runner $i" + continue + fi + ( + # Override ERR trap in subshell to prevent global side effects + trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR + configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "$repo" "$INSTANCE_ID" "$runner_grace_period" "$runner_initial_grace_period" + echo $? > /tmp/runner-$i-status + ) & + pids+=($!) + log "Started configuration for runner $i (PID: ${pids[-1]})" +done + +# Wait for all background jobs to complete +log "Waiting for all runner configurations to complete..." +failed=0 +succeeded=0 +for i in ${!pids[@]}; do + wait ${pids[$i]} + if [ -f /tmp/runner-$i-status ]; then + status=$(cat /tmp/runner-$i-status) + rm -f /tmp/runner-$i-status + if [ "$status" != "0" ]; then + log_error "Runner $i configuration failed" + failed=$((failed + 1)) + else + succeeded=$((succeeded + 1)) + fi + fi +done + +# Allow partial success - only terminate if ALL runners failed +if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then + terminate_instance "All runners failed to register" +elif [ $failed -gt 0 ]; then + log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." +fi + +if [ $succeeded -gt 0 ]; then + log "$succeeded runner(s) registered and started successfully" + touch $V-registered +else + log_error "No runners registered successfully" +fi + +# Kill registration watchdog now that runners are registered +if [ -f $V-watchdog.pid ]; then + WATCHDOG_PID=$(cat $V-watchdog.pid) + kill $WATCHDOG_PID 2>/dev/null || true + rm -f $V-watchdog.pid +fi + +# Final setup - ensure runner directories are accessible for debugging +touch $V-started +chmod o+x $homedir +for RUNNER_DIR in $homedir/runner-*; do + [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" +done + +log "Runner setup complete" diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 8004144..6799273 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -3,6 +3,7 @@ from os import environ from string import Template import json +import subprocess import boto3 from botocore.exceptions import ClientError @@ -14,6 +15,58 @@ from ec2_gha.defaults import AUTO, RUNNER_REGISTRATION_TIMEOUT +def resolve_ref_to_sha(ref: str) -> str: + """Resolve a Git ref (branch/tag/SHA) to a commit SHA using local git. + + Parameters + ---------- + ref : str + The Git ref to resolve (branch name, tag, or SHA) + + Returns + ------- + str + The commit SHA + + Raises + ------ + RuntimeError + If the ref cannot be resolved to a SHA + """ + # Handle Docker container ownership issues by marking directory as safe + # This is needed when running in GitHub Actions Docker containers where + # the workspace is owned by a different user than the container user + subprocess.run( + ['git', 'config', '--global', '--add', 'safe.directory', '/github/workspace'], + capture_output=True, + text=True, + check=True # Fail if this doesn't work - we need it for the next command + ) + + try: + # Use git rev-parse to resolve the ref to a SHA + # This works for branches, tags, and SHAs (returns SHAs unchanged) + result = subprocess.run( + ['git', 'rev-parse', ref], + capture_output=True, + text=True, + check=True + ) + sha = result.stdout.strip() + if sha: + # Only print if we actually resolved something (not just returned a SHA) + if sha != ref: + print(f"Resolved action_ref '{ref}' to SHA: {sha}") + return sha + else: + raise RuntimeError(f"git rev-parse returned empty output for ref '{ref}'") + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Failed to resolve action_ref '{ref}' to SHA. " + f"Error: {e.stderr or str(e)}" + ) + + @dataclass class StartAWS(CreateCloudInstance): """Class to start GitHub Actions runners on AWS. @@ -250,19 +303,6 @@ def _build_user_data(self, **kwargs) -> str: # Ensure instance_name has a default value kwargs.setdefault('instance_name', '') - # Load shared functions script - not a template, just include as-is - shared_functions_file = importlib.resources.files("ec2_gha").joinpath("templates/shared-functions.sh") - with shared_functions_file.open() as f: - shared_functions_content = f.read() - - # Strip the shebang line from shared functions since it will be embedded - shared_functions_lines = shared_functions_content.split('\n') - if shared_functions_lines[0].startswith('#!'): - shared_functions_content = '\n'.join(shared_functions_lines[1:]) - - # Add shared functions as-is to the main template kwargs - kwargs['shared_functions'] = shared_functions_content - template = importlib.resources.files("ec2_gha").joinpath("templates/user-script.sh.templ") with template.open() as f: template_content = f.read() @@ -271,22 +311,13 @@ def _build_user_data(self, **kwargs) -> str: parsed = Template(template_content) runner_script = parsed.substitute(**kwargs) - # Strip comment lines to save space (but keep shebang lines) - lines = runner_script.split('\n') - filtered_lines = [] - for line in lines: - stripped = line.strip() - # Keep shebang, empty lines, and non-comment lines - if not stripped or stripped.startswith('#!') or not stripped.startswith('#'): - filtered_lines.append(line) - - runner_script = '\n'.join(filtered_lines) - - # Log the final size + # Log the final size for informational purposes script_size = len(runner_script) print(f"UserData size: {script_size} bytes ({script_size/16384*100:.1f}% of 16KB limit)") return runner_script + except KeyError as e: + raise ValueError(f"Missing required template parameter: {e}") from e except Exception as e: raise Exception("Error parsing user data template") from e @@ -328,6 +359,7 @@ def _modify_root_disk_size(self, client, params: dict) -> dict: raise e return params + def create_instances(self) -> dict[str, str]: """Create instances on AWS. @@ -389,7 +421,14 @@ def create_instances(self) -> dict[str, str]: name_template = Template(name_pattern) instance_name_value = name_template.safe_substitute(**template_vars) + # Resolve action_ref to a SHA for security and consistency + action_ref = environ.get("INPUT_ACTION_REF") + if not action_ref: + raise ValueError("action_ref is required but was not provided. Check that runner.yml passes it correctly.") + action_sha = resolve_ref_to_sha(action_ref) + user_data_params = { + "action_sha": action_sha, # The resolved SHA "cloudwatch_logs_group": self.cloudwatch_logs_group, "debug": self.debug, "github_workflow": environ.get("GITHUB_WORKFLOW", ""), diff --git a/src/ec2_gha/templates/user-script.sh.templ b/src/ec2_gha/templates/user-script.sh.templ index 04c13cf..b360a55 100644 --- a/src/ec2_gha/templates/user-script.sh.templ +++ b/src/ec2_gha/templates/user-script.sh.templ @@ -1,410 +1,65 @@ #!/bin/bash set -e -# Enable debug tracing to a file for troubleshooting -exec 2> >(tee -a /var/log/runner-debug.log >&2) - -# Set debug variable for use in shared functions -debug="$debug" - -# Conditionally enable debug mode -[ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ] && set -x - -# Determine home directory early since it's needed by shared functions -homedir="$homedir" -if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then - # Try to find the default non-root user's home directory - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$$user" &>/dev/null; then - homedir="/home/$$user" - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $$homedir" | tee -a /var/log/runner-setup.log - break - fi - done - - # Fallback if no standard user found - if [ -z "$$homedir" ] || [ "$$homedir" = "AUTO" ]; then - homedir=$$(getent passwd | awk -F: '$$3 >= 1000 && $$3 < 65534 && $$6 ~ /^\\/home\\// {print $$6}' | while read dir; do - if [ -d "$$dir" ]; then - echo "$$dir" - break - fi - done) - if [ -z "$$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $$homedir" | tee -a /var/log/runner-setup.log - else - owner=$$(stat -c "%U" "$$homedir" 2>/dev/null || stat -f "%Su" "$$homedir" 2>/dev/null) - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $$homedir (owner: $$owner)" | tee -a /var/log/runner-setup.log - fi - fi -else - echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $$homedir" | tee -a /var/log/runner-setup.log -fi -export homedir - -# Set common paths -B=/usr/local/bin -V=/var/run/github-runner - -# Write shared functions that will be used by multiple scripts -# First write the variables that need template expansion -cat > $$B/runner-common.sh << EOSF -# Auto-generated shared functions and variables -# Set homedir for scripts that source this file -homedir="$$homedir" -debug="$$debug" -export homedir debug - -EOSF - -# Then append the shared functions using a quoted here-doc to prevent variable expansion -cat >> $$B/runner-common.sh << 'EOSF' -$shared_functions -EOSF - -chmod +x $$B/runner-common.sh -source $$B/runner-common.sh - -logger "EC2-GHA: Starting userdata script" -trap 'logger "EC2-GHA: Script failed at line $$LINENO with exit code $$?"' ERR -trap 'terminate_instance "Setup script failed with error on line $$LINENO"' ERR -# Handle watchdog termination signal -trap 'if [ -f $$V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM - -# Set up registration timeout failsafe - terminate if runner doesn't register in time -REGISTRATION_TIMEOUT="$runner_registration_timeout" -# Validate timeout is a number, default to 300 if not -if ! [[ "$$REGISTRATION_TIMEOUT" =~ ^[0-9]+$$ ]]; then - logger "EC2-GHA: Invalid timeout '$$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 -fi -logger "EC2-GHA: Registration timeout set to $$REGISTRATION_TIMEOUT seconds" -# Create a marker file for watchdog termination request -touch $$V-watchdog-active -( - log "Watchdog: Starting $$REGISTRATION_TIMEOUT second timeout" - sleep $$REGISTRATION_TIMEOUT - if [ ! -f $$V-registered ]; then - log "Watchdog: Registration marker not found after timeout" - # Signal main process to terminate instead of doing it directly - touch $$V-watchdog-terminate - # Kill the main script process to trigger its TERM trap - kill -TERM $$$$ 2>/dev/null || true +# Essential variables from template substitution +export debug="$debug" +export homedir="$homedir" +export repo="$repo" +export runner_tokens="$runner_tokens" +export runner_labels="$runner_labels" +export cloudwatch_logs_group="$cloudwatch_logs_group" +export runner_grace_period="$runner_grace_period" +export runner_initial_grace_period="$runner_initial_grace_period" +export runner_poll_interval="$runner_poll_interval" +export runner_registration_timeout="$runner_registration_timeout" +export max_instance_lifetime="$max_instance_lifetime" +export runners_per_instance="$runners_per_instance" +export runner_release="$runner_release" +export ssh_pubkey="$ssh_pubkey" +export instance_name="$instance_name" +export action_sha="$action_sha" + +# Custom userdata from user (if any) +export userdata="$userdata" +export script="$script" + +# Log prefixes +export log_prefix_job_started="$log_prefix_job_started" +export log_prefix_job_completed="$log_prefix_job_completed" + +# Fetch and execute the main script from GitHub +# action_sha has already been resolved from action_ref in Python for security and consistency +SCRIPT_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/$${action_sha}/src/ec2_gha/scripts/runner-setup.sh" +echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Fetching main script from: $$SCRIPT_URL (SHA: $${action_sha})" | tee -a /var/log/runner-setup.log + +# Try to download with retries +for i in {1..5}; do + if curl -sSL "$$SCRIPT_URL" -o /tmp/runner-setup.sh; then + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $$i)" | tee -a /var/log/runner-setup.log + break + elif wget -q "$$SCRIPT_URL" -O /tmp/runner-setup.sh; then + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $$i)" | tee -a /var/log/runner-setup.log + break else - log "Watchdog: Registration marker found, exiting normally" - fi - rm -f $$V-watchdog-active -) & -REGISTRATION_WATCHDOG_PID=$$! -log "Watchdog PID: $$REGISTRATION_WATCHDOG_PID" -echo $$REGISTRATION_WATCHDOG_PID > $$V-watchdog.pid - -# Run any custom user data script provided by the user -$userdata - -exec >> /var/log/runner-setup.log 2>&1 -log "Starting runner setup" - -# Fetch instance metadata for labeling and logging -INSTANCE_TYPE=$$(get_metadata "instance-type") -INSTANCE_ID=$$(get_metadata "instance-id") -REGION=$$(get_metadata "placement/region") -AZ=$$(get_metadata "placement/availability-zone") -log "Instance metadata: Type=$${INSTANCE_TYPE} ID=$${INSTANCE_ID} Region=$${REGION} AZ=$${AZ}" - -# Set up maximum lifetime timeout - instance will terminate after this time regardless of job status -MAX_LIFETIME_MINUTES=$max_instance_lifetime -log "Setting up maximum lifetime timeout: $${MAX_LIFETIME_MINUTES} minutes" -# Use ; instead of && so shutdown runs even if echo fails (e.g., disk full) -# Try multiple shutdown methods as fallbacks -nohup bash -c " - sleep $${MAX_LIFETIME_MINUTES}m - echo '[$$(date)] Maximum lifetime reached' 2>/dev/null || true - # Try normal shutdown - shutdown -h now 2>/dev/null || { - # If shutdown fails, try halt - halt -f 2>/dev/null || { - # If halt fails, try sysrq if available (Linux only) - if [ -w /proc/sysrq-trigger ]; then - echo 1 > /proc/sys/kernel/sysrq 2>/dev/null - echo o > /proc/sysrq-trigger 2>/dev/null - fi - # Last resort: force immediate reboot - reboot -f 2>/dev/null || true - } - } -" > /var/log/max-lifetime.log 2>&1 & - -# Configure CloudWatch Logs if a log group is specified -if [ "$cloudwatch_logs_group" != "" ]; then - log "Installing CloudWatch agent" - # Use a subshell to prevent CloudWatch failures from stopping the entire script - ( - # Detect package manager and install CloudWatch agent - if command -v dpkg >/dev/null 2>&1; then - # Debian/Ubuntu - log "Detected dpkg-based system" - wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb - elif command -v rpm >/dev/null 2>&1; then - # RHEL/CentOS/Amazon Linux - log "Detected rpm-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm - rpm -U ./amazon-cloudwatch-agent.rpm - rm amazon-cloudwatch-agent.rpm - else - log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" - fi - # Build CloudWatch config with factored strings - P='"file_path":' - G=',"log_group_name":"$cloudwatch_logs_group","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$$homedir" - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF -{"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ -{$$P"/var/log/runner-setup.log"$${G}runner-setup$$Z, -{$$P"/var/log/runner-debug.log"$${G}runner-debug$$Z, -{$$P"/tmp/job-started-hook.log"$${G}job-started$$Z, -{$$P"/tmp/job-completed-hook.log"$${G}job-completed$$Z, -{$$P"/tmp/termination-check.log"$${G}termination$$Z, -{$$P"/tmp/runner-*-config.log"$${G}runner-config$$Z, -{$$P"$$H/_diag/Runner_**.log"$${G}runner-diag$$Z, -{$$P"$$H/_diag/Worker_**.log"$${G}worker-diag$$Z -]}}}} -EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s - log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" -fi - -# Configure SSH access if public key provided (useful for debugging) -if [ -n "$ssh_pubkey" ]; then - log "Configuring SSH access" - # Determine the default user based on the home directory owner - DEFAULT_USER=$$(stat -c "%U" "$$homedir" 2>/dev/null || echo "root") - mkdir -p "$$homedir/.ssh" - chmod 700 "$$homedir/.ssh" - echo "$ssh_pubkey" >> "$$homedir/.ssh/authorized_keys" - chmod 600 "$$homedir/.ssh/authorized_keys" - if [ "$$DEFAULT_USER" != "root" ]; then - chown -R "$$DEFAULT_USER:$$DEFAULT_USER" "$$homedir/.ssh" - fi - log "SSH key added for user $$DEFAULT_USER" -fi - -log "Working directory: $$homedir" -cd "$$homedir" - -# Run any pre-runner script provided by the user -echo "$script" > pre-runner-script.sh -log "Running pre-runner script" -source pre-runner-script.sh -export RUNNER_ALLOW_RUNASROOT=1 - -# Number of runners to configure on this instance -RUNNERS_PER_INSTANCE=$runners_per_instance - -# Download GitHub Actions runner binary -ARCH=$$(uname -m) -if [ "$$ARCH" = "aarch64" ] || [ "$$ARCH" = "arm64" ]; then - RUNNER_URL=$$(echo "$runner_release" | sed 's/x64/arm64/g') - log "ARM detected, using: $$RUNNER_URL" -else - RUNNER_URL="$runner_release" - log "x64 detected, using: $$RUNNER_URL" -fi - -if command -v curl >/dev/null 2>&1; then - curl -L $$RUNNER_URL -o /tmp/runner.tar.gz -elif command -v wget >/dev/null 2>&1; then - wget -q $$RUNNER_URL -O /tmp/runner.tar.gz -else - log_error "Neither curl nor wget found. Cannot download runner." - terminate_instance "No download tool available" -fi -log "Downloaded runner binary" - -# Create job tracking scripts - these are called by GitHub runner hooks -# job-started-hook.sh is called when a job starts -cat > $$B/job-started-hook.sh << EOFS -#!/bin/bash -exec >> /tmp/job-started-hook.log 2>&1 -I="\$${RUNNER_INDEX:-0}" -echo "[\$$(date)] Runner-\$$I: ${log_prefix_job_started} \$${GITHUB_JOB}" -mkdir -p $$V-jobs -echo '{"status":"running","runner":"'\$$I'"}' > $$V-jobs/\$${GITHUB_RUN_ID}-\$${GITHUB_JOB}-\$$I.job -touch $$V-last-activity $$V-has-run-job -EOFS - -# job-completed-hook.sh is called when a job completes -cat > $$B/job-completed-hook.sh << EOFC -#!/bin/bash -exec >> /tmp/job-completed-hook.log 2>&1 -I="\$${RUNNER_INDEX:-0}" -echo "[\$$(date)] Runner-\$$I: ${log_prefix_job_completed} \$${GITHUB_JOB}" -rm -f $$V-jobs/\$${GITHUB_RUN_ID}-\$${GITHUB_JOB}-\$$I.job -touch $$V-last-activity -EOFC - -# check-runner-termination.sh is called periodically to check if the instance should terminate -cat > $$B/check-runner-termination.sh << EOFT -#!/bin/bash -exec >> /tmp/termination-check.log 2>&1 -source $$B/runner-common.sh -A="$$V-last-activity" -J="$$V-jobs" -H="$$V-has-run-job" - -# Check if any runners are actually running -RUNNER_PROCS=\$$(pgrep -f "Runner.Listener" | wc -l) -if [ \$$RUNNER_PROCS -eq 0 ]; then - # No runner processes, check if we have stale job files - if ls \$$J/*.job 2>/dev/null | grep -q .; then - log "WARNING: Found job files but no runner processes - cleaning up stale jobs" - rm -f \$$J/*.job - fi -fi - -[ ! -f "\$$A" ] && touch "\$$A" -L=\$$(stat -c %Y "\$$A" 2>/dev/null || echo 0) -N=\$$(date +%s) -I=\$$((N-L)) -[ -f "\$$H" ] && G=\$${RUNNER_GRACE_PERIOD:-60} || G=\$${RUNNER_INITIAL_GRACE_PERIOD:-180} -R=\$$(grep -l '"status":"running"' \$$J/*.job 2>/dev/null | wc -l || echo 0) -if [ \$$R -eq 0 ] && [ \$$I -gt \$$G ]; then - log "TERMINATING: idle \$$I > grace \$$G" - deregister_all_runners - flush_cloudwatch_logs - debug_sleep_and_shutdown -else - [ \$$R -gt 0 ] && log "\$$R job(s) running" || log "Idle \$$I/\$$G sec" -fi -EOFT - -chmod +x $$B/job-started-hook.sh $$B/job-completed-hook.sh $$B/check-runner-termination.sh - -# Set up job tracking directory -mkdir -p $$V-jobs -touch $$V-last-activity - -# Set up periodic termination check using systemd -cat > /etc/systemd/system/runner-termination-check.service << EOF -[Unit] -Description=Check GitHub runner termination conditions -After=network.target -[Service] -Type=oneshot -Environment="RUNNER_GRACE_PERIOD=$runner_grace_period" -Environment="RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" -ExecStart=$$B/check-runner-termination.sh -EOF - -cat > /etc/systemd/system/runner-termination-check.timer << EOF -[Unit] -Description=Periodic GitHub runner termination check -Requires=runner-termination-check.service -[Timer] -OnBootSec=60s -OnUnitActiveSec=${runner_poll_interval}s -[Install] -WantedBy=timers.target -EOF - -systemctl daemon-reload -systemctl enable runner-termination-check.timer -systemctl start runner-termination-check.timer - -# Build metadata labels (these will be added to the runner labels) -METADATA_LABELS=",$${INSTANCE_ID},$${INSTANCE_TYPE}" -# Add instance name as a label if provided -if [ -n "$instance_name" ]; then - INSTANCE_NAME_LABEL=$$(echo "$instance_name" | tr ' /' '-' | tr -cd '[:alnum:]-_#') - METADATA_LABELS="$${METADATA_LABELS},$${INSTANCE_NAME_LABEL}" -fi - -log "Setting up $$RUNNERS_PER_INSTANCE runner(s)" - -# Export functions for subprocesses (variables already exported from runner-common.sh) -export -f configure_runner -export -f log -export -f log_error -export -f get_metadata -export -f flush_cloudwatch_logs -export -f deregister_all_runners -export -f debug_sleep_and_shutdown -export -f wait_for_dpkg_lock - -# Parse space-delimited tokens and pipe-delimited labels -IFS=' ' read -ra tokens <<< "$runner_tokens" -IFS='|' read -ra labels <<< "$runner_labels" - -num_runners=$${#tokens[@]} -log "Configuring $$num_runners runner(s) in parallel" - -# Start configuration for each runner in parallel -pids=() -for i in $${!tokens[@]}; do - token=$${tokens[$$i]} - label=$${labels[$$i]:-} - if [ -z "$$token" ]; then - log_error "No token for runner $$i" - continue + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Failed to download script (attempt $$i), retrying..." | tee -a /var/log/runner-setup.log + sleep 2 fi - ( - # Override ERR trap in subshell to prevent global side effects - trap 'echo "Subshell error on line $$LINENO" >&2; exit 1' ERR - configure_runner $$i "$$token" "$${label}$$METADATA_LABELS" "$$homedir" "$repo" "$$INSTANCE_ID" "$runner_grace_period" "$runner_initial_grace_period" - echo $$? > /tmp/runner-$$i-status - ) & - pids+=($$!) - log "Started configuration for runner $$i (PID: $${pids[-1]})" -done -# Wait for all background jobs to complete -log "Waiting for all runner configurations to complete..." -failed=0 -succeeded=0 -for i in $${!pids[@]}; do - wait $${pids[$$i]} - if [ -f /tmp/runner-$$i-status ]; then - status=$$(cat /tmp/runner-$$i-status) - rm -f /tmp/runner-$$i-status - if [ "$$status" != "0" ]; then - log_error "Runner $$i configuration failed" - failed=$$((failed + 1)) - else - succeeded=$$((succeeded + 1)) - fi + if [ $$i -eq 5 ]; then + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download runner setup script after 5 attempts" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi done -# Allow partial success - only terminate if ALL runners failed -if [ $$succeeded -eq 0 ] && [ $$failed -gt 0 ]; then - terminate_instance "All runners failed to register" -elif [ $$failed -gt 0 ]; then - log "WARNING: $$failed runner(s) failed, but $$succeeded succeeded. Continuing with partial capacity." -fi - -if [ $$succeeded -gt 0 ]; then - log "$$succeeded runner(s) registered and started successfully" - touch $$V-registered -else - log_error "No runners registered successfully" +# Verify we got something +if [ ! -s /tmp/runner-setup.sh ]; then + echo "[$$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Downloaded script is empty" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi -# Kill registration watchdog now that runners are registered -if [ -f $$V-watchdog.pid ]; then - WATCHDOG_PID=$$(cat $$V-watchdog.pid) - kill $$WATCHDOG_PID 2>/dev/null || true - rm -f $$V-watchdog.pid -fi - -# Final setup - ensure runner directories are accessible for debugging -touch $$V-started -chmod o+x $$homedir -for RUNNER_DIR in $$homedir/runner-*; do - [ -d "$$RUNNER_DIR/_diag" ] && chmod 755 "$$RUNNER_DIR/_diag" -done +# Make it executable and run it +chmod +x /tmp/runner-setup.sh +echo "[$$(date '+%Y-%m-%d %H:%M:%S')] Executing runner setup script" | tee -a /var/log/runner-setup.log +exec /tmp/runner-setup.sh diff --git a/tests/__snapshots__/test_start.ambr b/tests/__snapshots__/test_start.ambr index 50c7968..b53a1fd 100644 --- a/tests/__snapshots__/test_start.ambr +++ b/tests/__snapshots__/test_start.ambr @@ -44,517 +44,68 @@ #!/bin/bash set -e - exec 2> >(tee -a /var/log/runner-debug.log >&2) - - debug="" - - [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x - - homedir="/home/ec2-user" - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log - fi - fi - else - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log - fi - export homedir - - B=/usr/local/bin - V=/var/run/github-runner - - cat > $B/runner-common.sh << EOSF - homedir="$homedir" - debug="$debug" - export homedir debug - - EOSF - - cat >> $B/runner-common.sh << 'EOSF' - - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } - log_error() { log "ERROR: $1" >&2; } - - dn=/dev/null - - wait_for_dpkg_lock() { - local t=120 - local L=/var/lib/dpkg/lock - while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do - if [ $t -le 0 ]; then - log "WARNING: dpkg lock t, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($t seconds remaining)" - sleep 5 - t=$((t - 5)) - done - } - - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true - fi - } - - get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - fi - return 0 # Always return success to avoid set -e issues - } - - deregister_all_runners() { - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in $RUNNER_DIR" - cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true - sleep 1 - if [ -f "$RUNNER_DIR/.runner-token" ]; then - TOKEN=$(cat "$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 - log "Deregistration exit: $?" - fi - fi - done - } - - debug_sleep_and_shutdown() { - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") - local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) - log "SSH into instance with: ssh ${ssh_user}@${public_ip}" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 600 - log "Debug period ended, shutting down" - else - log "Shutting down immediately (debug mode not enabled)" - fi - shutdown -h now - } - - terminate_instance() { - local reason="$1" - local instance_id=$(get_metadata "instance-id") - - echo "========================================" | tee -a /var/log/runner-setup.log - log "FATAL ERROR DETECTED" - log "Reason: $reason" - log "Instance: $instance_id" - log "Script location: $(pwd)" - log "User: $(whoami)" - log "Debug trace available in: /var/log/runner-debug.log" - echo "========================================" | tee -a /var/log/runner-setup.log - - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi - - flush_cloudwatch_logs - debug_sleep_and_shutdown - exit 1 - } - - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - tar -xzf /tmp/runner.tar.gz - - if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then - log "Dependencies exist, skipping install" - else - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >$dn 2>&1 - local deps_result=$? - set -e - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing manually..." - if command -v dnf >$dn 2>&1; then - sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true - elif command -v yum >$dn 2>&1; then - sudo yum install -y libicu >$dn 2>&1 || true - elif command -v apt-get >$dn 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >$dn 2>&1 || true - sudo apt-get install -y libicu-dev >$dn 2>&1 || true - fi - fi - fi - fi - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } - - EOSF - - chmod +x $B/runner-common.sh - source $B/runner-common.sh - - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM - - REGISTRATION_TIMEOUT="300" - if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 - fi - logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" - touch $V-watchdog-active - ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f $V-registered ]; then - log "Watchdog: Registration marker not found after timeout" - touch $V-watchdog-terminate - kill -TERM $$ 2>/dev/null || true + # Essential variables from template substitution + export debug="" + export homedir="/home/ec2-user" + export repo="omsf-eco-infra/awsinfratesting" + export runner_tokens="test" + export runner_labels="label" + export cloudwatch_logs_group="" + export runner_grace_period="61" + export runner_initial_grace_period="181" + export runner_poll_interval="11" + export runner_registration_timeout="300" + export max_instance_lifetime="360" + export runners_per_instance="1" + export runner_release="test.tar.gz" + export ssh_pubkey="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" + export instance_name="" + export action_sha="abc123def456789012345678901234567890abcd" + + # Custom userdata from user (if any) + export userdata="" + export script="echo 'Hello, World!'" + + # Log prefixes + export log_prefix_job_started="Job started:" + export log_prefix_job_completed="Job completed:" + + # Fetch and execute the main script from GitHub + # action_sha has already been resolved from action_ref in Python for security and consistency + SCRIPT_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/${action_sha}/src/ec2_gha/scripts/runner-setup.sh" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching main script from: $SCRIPT_URL (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log + + # Try to download with retries + for i in {1..5}; do + if curl -sSL "$SCRIPT_URL" -o /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break + elif wget -q "$SCRIPT_URL" -O /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break else - log "Watchdog: Registration marker found, exiting normally" - fi - rm -f $V-watchdog-active - ) & - REGISTRATION_WATCHDOG_PID=$! - log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid - - - - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - - INSTANCE_TYPE=$(get_metadata "instance-type") - INSTANCE_ID=$(get_metadata "instance-id") - REGION=$(get_metadata "placement/region") - AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - - MAX_LIFETIME_MINUTES=360 - log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c " - sleep ${MAX_LIFETIME_MINUTES}m - echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true - shutdown -h now 2>/dev/null || { - halt -f 2>/dev/null || { - if [ -w /proc/sysrq-trigger ]; then - echo 1 > /proc/sys/kernel/sysrq 2>/dev/null - echo o > /proc/sysrq-trigger 2>/dev/null - fi - reboot -f 2>/dev/null || true - } - } - " > /var/log/max-lifetime.log 2>&1 & - - if [ "" != "" ]; then - log "Installing CloudWatch agent" - ( - if command -v dpkg >/dev/null 2>&1; then - log "Detected dpkg-based system" - wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb - elif command -v rpm >/dev/null 2>&1; then - log "Detected rpm-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm - rpm -U ./amazon-cloudwatch-agent.rpm - rm amazon-cloudwatch-agent.rpm - else - log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" - fi - P='"file_path":' - G=',"log_group_name":"","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$homedir" - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {$P"/var/log/runner-setup.log"${G}runner-setup$Z, - {$P"/var/log/runner-debug.log"${G}runner-debug$Z, - {$P"/tmp/job-started-hook.log"${G}job-started$Z, - {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, - {$P"/tmp/termination-check.log"${G}termination$Z, - {$P"/tmp/runner-*-config.log"${G}runner-config$Z, - {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, - {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z - ]}}}} - EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s - log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" - fi - - if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - mkdir -p "$homedir/.ssh" - chmod 700 "$homedir/.ssh" - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi - log "SSH key added for user $DEFAULT_USER" - fi - - log "Working directory: $homedir" - cd "$homedir" - - echo "echo 'Hello, World!'" > pre-runner-script.sh - log "Running pre-runner script" - source pre-runner-script.sh - export RUNNER_ALLOW_RUNASROOT=1 - - RUNNERS_PER_INSTANCE=1 - - ARCH=$(uname -m) - if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" - else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" - fi - - if command -v curl >/dev/null 2>&1; then - curl -L $RUNNER_URL -o /tmp/runner.tar.gz - elif command -v wget >/dev/null 2>&1; then - wget -q $RUNNER_URL -O /tmp/runner.tar.gz - else - log_error "Neither curl nor wget found. Cannot download runner." - terminate_instance "No download tool available" - fi - log "Downloaded runner binary" - - cat > $B/job-started-hook.sh << EOFS - #!/bin/bash - exec >> /tmp/job-started-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" - mkdir -p $V-jobs - echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity $V-has-run-job - EOFS - - cat > $B/job-completed-hook.sh << EOFC - #!/bin/bash - exec >> /tmp/job-completed-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" - rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity - EOFC - - cat > $B/check-runner-termination.sh << EOFT - #!/bin/bash - exec >> /tmp/termination-check.log 2>&1 - source $B/runner-common.sh - A="$V-last-activity" - J="$V-jobs" - H="$V-has-run-job" - - RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) - if [ \$RUNNER_PROCS -eq 0 ]; then - if ls \$J/*.job 2>/dev/null | grep -q .; then - log "WARNING: Found job files but no runner processes - cleaning up stale jobs" - rm -f \$J/*.job - fi - fi - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - deregister_all_runners - flush_cloudwatch_logs - debug_sleep_and_shutdown - else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" - fi - EOFT - - chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - - mkdir -p $V-jobs - touch $V-last-activity - - cat > /etc/systemd/system/runner-termination-check.service << EOF - [Unit] - Description=Check GitHub runner termination conditions - After=network.target - [Service] - Type=oneshot - Environment="RUNNER_GRACE_PERIOD=61" - Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=$B/check-runner-termination.sh - EOF - - cat > /etc/systemd/system/runner-termination-check.timer << EOF - [Unit] - Description=Periodic GitHub runner termination check - Requires=runner-termination-check.service - [Timer] - OnBootSec=60s - OnUnitActiveSec=11s - [Install] - WantedBy=timers.target - EOF - - systemctl daemon-reload - systemctl enable runner-termination-check.timer - systemctl start runner-termination-check.timer - - METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" - if [ -n "" ]; then - INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') - METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" - fi - - log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - - export -f configure_runner - export -f log - export -f log_error - export -f get_metadata - export -f flush_cloudwatch_logs - export -f deregister_all_runners - export -f debug_sleep_and_shutdown - export -f wait_for_dpkg_lock - - IFS=' ' read -ra tokens <<< "test" - IFS='|' read -ra labels <<< "label" - - num_runners=${#tokens[@]} - log "Configuring $num_runners runner(s) in parallel" - - pids=() - for i in ${!tokens[@]}; do - token=${tokens[$i]} - label=${labels[$i]:-} - if [ -z "$token" ]; then - log_error "No token for runner $i" - continue + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Failed to download script (attempt $i), retrying..." | tee -a /var/log/runner-setup.log + sleep 2 fi - ( - trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR - configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" - echo $? > /tmp/runner-$i-status - ) & - pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" - done - log "Waiting for all runner configurations to complete..." - failed=0 - succeeded=0 - for i in ${!pids[@]}; do - wait ${pids[$i]} - if [ -f /tmp/runner-$i-status ]; then - status=$(cat /tmp/runner-$i-status) - rm -f /tmp/runner-$i-status - if [ "$status" != "0" ]; then - log_error "Runner $i configuration failed" - failed=$((failed + 1)) - else - succeeded=$((succeeded + 1)) - fi + if [ $i -eq 5 ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download runner setup script after 5 attempts" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi done - if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then - terminate_instance "All runners failed to register" - elif [ $failed -gt 0 ]; then - log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." - fi - - if [ $succeeded -gt 0 ]; then - log "$succeeded runner(s) registered and started successfully" - touch $V-registered - else - log_error "No runners registered successfully" - fi - - if [ -f $V-watchdog.pid ]; then - WATCHDOG_PID=$(cat $V-watchdog.pid) - kill $WATCHDOG_PID 2>/dev/null || true - rm -f $V-watchdog.pid + # Verify we got something + if [ ! -s /tmp/runner-setup.sh ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Downloaded script is empty" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi - touch $V-started - chmod o+x $homedir - for RUNNER_DIR in $homedir/runner-*; do - [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" - done - + # Make it executable and run it + chmod +x /tmp/runner-setup.sh + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Executing runner setup script" | tee -a /var/log/runner-setup.log + exec /tmp/runner-setup.sh ''', }) # --- @@ -599,517 +150,68 @@ #!/bin/bash set -e - exec 2> >(tee -a /var/log/runner-debug.log >&2) - - debug="" - - [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x - - homedir="/home/ec2-user" - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log - fi - fi - else - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log - fi - export homedir - - B=/usr/local/bin - V=/var/run/github-runner - - cat > $B/runner-common.sh << EOSF - homedir="$homedir" - debug="$debug" - export homedir debug - - EOSF - - cat >> $B/runner-common.sh << 'EOSF' - - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } - log_error() { log "ERROR: $1" >&2; } - - dn=/dev/null - - wait_for_dpkg_lock() { - local t=120 - local L=/var/lib/dpkg/lock - while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do - if [ $t -le 0 ]; then - log "WARNING: dpkg lock t, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($t seconds remaining)" - sleep 5 - t=$((t - 5)) - done - } - - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true - fi - } - - get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - fi - return 0 # Always return success to avoid set -e issues - } - - deregister_all_runners() { - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in $RUNNER_DIR" - cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true - sleep 1 - if [ -f "$RUNNER_DIR/.runner-token" ]; then - TOKEN=$(cat "$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 - log "Deregistration exit: $?" - fi - fi - done - } - - debug_sleep_and_shutdown() { - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") - local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) - log "SSH into instance with: ssh ${ssh_user}@${public_ip}" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 600 - log "Debug period ended, shutting down" - else - log "Shutting down immediately (debug mode not enabled)" - fi - shutdown -h now - } - - terminate_instance() { - local reason="$1" - local instance_id=$(get_metadata "instance-id") - - echo "========================================" | tee -a /var/log/runner-setup.log - log "FATAL ERROR DETECTED" - log "Reason: $reason" - log "Instance: $instance_id" - log "Script location: $(pwd)" - log "User: $(whoami)" - log "Debug trace available in: /var/log/runner-debug.log" - echo "========================================" | tee -a /var/log/runner-setup.log - - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi - - flush_cloudwatch_logs - debug_sleep_and_shutdown - exit 1 - } - - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - tar -xzf /tmp/runner.tar.gz - - if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then - log "Dependencies exist, skipping install" - else - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >$dn 2>&1 - local deps_result=$? - set -e - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing manually..." - if command -v dnf >$dn 2>&1; then - sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true - elif command -v yum >$dn 2>&1; then - sudo yum install -y libicu >$dn 2>&1 || true - elif command -v apt-get >$dn 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >$dn 2>&1 || true - sudo apt-get install -y libicu-dev >$dn 2>&1 || true - fi - fi - fi - fi - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } - - EOSF - - chmod +x $B/runner-common.sh - source $B/runner-common.sh - - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM - - REGISTRATION_TIMEOUT="300" - if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 - fi - logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" - touch $V-watchdog-active - ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f $V-registered ]; then - log "Watchdog: Registration marker not found after timeout" - touch $V-watchdog-terminate - kill -TERM $$ 2>/dev/null || true + # Essential variables from template substitution + export debug="" + export homedir="/home/ec2-user" + export repo="omsf-eco-infra/awsinfratesting" + export runner_tokens="test" + export runner_labels="label" + export cloudwatch_logs_group="" + export runner_grace_period="61" + export runner_initial_grace_period="181" + export runner_poll_interval="11" + export runner_registration_timeout="300" + export max_instance_lifetime="360" + export runners_per_instance="1" + export runner_release="test.tar.gz" + export ssh_pubkey="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" + export instance_name="" + export action_sha="abc123def456789012345678901234567890abcd" + + # Custom userdata from user (if any) + export userdata="" + export script="echo 'Hello, World!'" + + # Log prefixes + export log_prefix_job_started="Job started:" + export log_prefix_job_completed="Job completed:" + + # Fetch and execute the main script from GitHub + # action_sha has already been resolved from action_ref in Python for security and consistency + SCRIPT_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/${action_sha}/src/ec2_gha/scripts/runner-setup.sh" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching main script from: $SCRIPT_URL (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log + + # Try to download with retries + for i in {1..5}; do + if curl -sSL "$SCRIPT_URL" -o /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break + elif wget -q "$SCRIPT_URL" -O /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break else - log "Watchdog: Registration marker found, exiting normally" - fi - rm -f $V-watchdog-active - ) & - REGISTRATION_WATCHDOG_PID=$! - log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid - - - - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - - INSTANCE_TYPE=$(get_metadata "instance-type") - INSTANCE_ID=$(get_metadata "instance-id") - REGION=$(get_metadata "placement/region") - AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - - MAX_LIFETIME_MINUTES=360 - log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c " - sleep ${MAX_LIFETIME_MINUTES}m - echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true - shutdown -h now 2>/dev/null || { - halt -f 2>/dev/null || { - if [ -w /proc/sysrq-trigger ]; then - echo 1 > /proc/sys/kernel/sysrq 2>/dev/null - echo o > /proc/sysrq-trigger 2>/dev/null - fi - reboot -f 2>/dev/null || true - } - } - " > /var/log/max-lifetime.log 2>&1 & - - if [ "" != "" ]; then - log "Installing CloudWatch agent" - ( - if command -v dpkg >/dev/null 2>&1; then - log "Detected dpkg-based system" - wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb - elif command -v rpm >/dev/null 2>&1; then - log "Detected rpm-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm - rpm -U ./amazon-cloudwatch-agent.rpm - rm amazon-cloudwatch-agent.rpm - else - log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" - fi - P='"file_path":' - G=',"log_group_name":"","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$homedir" - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {$P"/var/log/runner-setup.log"${G}runner-setup$Z, - {$P"/var/log/runner-debug.log"${G}runner-debug$Z, - {$P"/tmp/job-started-hook.log"${G}job-started$Z, - {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, - {$P"/tmp/termination-check.log"${G}termination$Z, - {$P"/tmp/runner-*-config.log"${G}runner-config$Z, - {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, - {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z - ]}}}} - EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s - log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" - fi - - if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - mkdir -p "$homedir/.ssh" - chmod 700 "$homedir/.ssh" - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi - log "SSH key added for user $DEFAULT_USER" - fi - - log "Working directory: $homedir" - cd "$homedir" - - echo "echo 'Hello, World!'" > pre-runner-script.sh - log "Running pre-runner script" - source pre-runner-script.sh - export RUNNER_ALLOW_RUNASROOT=1 - - RUNNERS_PER_INSTANCE=1 - - ARCH=$(uname -m) - if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" - else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" - fi - - if command -v curl >/dev/null 2>&1; then - curl -L $RUNNER_URL -o /tmp/runner.tar.gz - elif command -v wget >/dev/null 2>&1; then - wget -q $RUNNER_URL -O /tmp/runner.tar.gz - else - log_error "Neither curl nor wget found. Cannot download runner." - terminate_instance "No download tool available" - fi - log "Downloaded runner binary" - - cat > $B/job-started-hook.sh << EOFS - #!/bin/bash - exec >> /tmp/job-started-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" - mkdir -p $V-jobs - echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity $V-has-run-job - EOFS - - cat > $B/job-completed-hook.sh << EOFC - #!/bin/bash - exec >> /tmp/job-completed-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" - rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity - EOFC - - cat > $B/check-runner-termination.sh << EOFT - #!/bin/bash - exec >> /tmp/termination-check.log 2>&1 - source $B/runner-common.sh - A="$V-last-activity" - J="$V-jobs" - H="$V-has-run-job" - - RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) - if [ \$RUNNER_PROCS -eq 0 ]; then - if ls \$J/*.job 2>/dev/null | grep -q .; then - log "WARNING: Found job files but no runner processes - cleaning up stale jobs" - rm -f \$J/*.job - fi - fi - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - deregister_all_runners - flush_cloudwatch_logs - debug_sleep_and_shutdown - else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" - fi - EOFT - - chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - - mkdir -p $V-jobs - touch $V-last-activity - - cat > /etc/systemd/system/runner-termination-check.service << EOF - [Unit] - Description=Check GitHub runner termination conditions - After=network.target - [Service] - Type=oneshot - Environment="RUNNER_GRACE_PERIOD=61" - Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=$B/check-runner-termination.sh - EOF - - cat > /etc/systemd/system/runner-termination-check.timer << EOF - [Unit] - Description=Periodic GitHub runner termination check - Requires=runner-termination-check.service - [Timer] - OnBootSec=60s - OnUnitActiveSec=11s - [Install] - WantedBy=timers.target - EOF - - systemctl daemon-reload - systemctl enable runner-termination-check.timer - systemctl start runner-termination-check.timer - - METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" - if [ -n "" ]; then - INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') - METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" - fi - - log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - - export -f configure_runner - export -f log - export -f log_error - export -f get_metadata - export -f flush_cloudwatch_logs - export -f deregister_all_runners - export -f debug_sleep_and_shutdown - export -f wait_for_dpkg_lock - - IFS=' ' read -ra tokens <<< "test" - IFS='|' read -ra labels <<< "label" - - num_runners=${#tokens[@]} - log "Configuring $num_runners runner(s) in parallel" - - pids=() - for i in ${!tokens[@]}; do - token=${tokens[$i]} - label=${labels[$i]:-} - if [ -z "$token" ]; then - log_error "No token for runner $i" - continue + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Failed to download script (attempt $i), retrying..." | tee -a /var/log/runner-setup.log + sleep 2 fi - ( - trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR - configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" - echo $? > /tmp/runner-$i-status - ) & - pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" - done - log "Waiting for all runner configurations to complete..." - failed=0 - succeeded=0 - for i in ${!pids[@]}; do - wait ${pids[$i]} - if [ -f /tmp/runner-$i-status ]; then - status=$(cat /tmp/runner-$i-status) - rm -f /tmp/runner-$i-status - if [ "$status" != "0" ]; then - log_error "Runner $i configuration failed" - failed=$((failed + 1)) - else - succeeded=$((succeeded + 1)) - fi + if [ $i -eq 5 ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download runner setup script after 5 attempts" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi done - if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then - terminate_instance "All runners failed to register" - elif [ $failed -gt 0 ]; then - log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." - fi - - if [ $succeeded -gt 0 ]; then - log "$succeeded runner(s) registered and started successfully" - touch $V-registered - else - log_error "No runners registered successfully" - fi - - if [ -f $V-watchdog.pid ]; then - WATCHDOG_PID=$(cat $V-watchdog.pid) - kill $WATCHDOG_PID 2>/dev/null || true - rm -f $V-watchdog.pid + # Verify we got something + if [ ! -s /tmp/runner-setup.sh ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Downloaded script is empty" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi - touch $V-started - chmod o+x $homedir - for RUNNER_DIR in $homedir/runner-*; do - [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" - done - + # Make it executable and run it + chmod +x /tmp/runner-setup.sh + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Executing runner setup script" | tee -a /var/log/runner-setup.log + exec /tmp/runner-setup.sh ''', }) # --- @@ -1118,517 +220,68 @@ #!/bin/bash set -e - exec 2> >(tee -a /var/log/runner-debug.log >&2) - - debug="" - - [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x - - homedir="/home/ec2-user" - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log - fi - fi - else - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log - fi - export homedir - - B=/usr/local/bin - V=/var/run/github-runner - - cat > $B/runner-common.sh << EOSF - homedir="$homedir" - debug="$debug" - export homedir debug - - EOSF - - cat >> $B/runner-common.sh << 'EOSF' - - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } - log_error() { log "ERROR: $1" >&2; } - - dn=/dev/null - - wait_for_dpkg_lock() { - local t=120 - local L=/var/lib/dpkg/lock - while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do - if [ $t -le 0 ]; then - log "WARNING: dpkg lock t, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($t seconds remaining)" - sleep 5 - t=$((t - 5)) - done - } - - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true - fi - } - - get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - fi - return 0 # Always return success to avoid set -e issues - } - - deregister_all_runners() { - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in $RUNNER_DIR" - cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true - sleep 1 - if [ -f "$RUNNER_DIR/.runner-token" ]; then - TOKEN=$(cat "$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 - log "Deregistration exit: $?" - fi - fi - done - } - - debug_sleep_and_shutdown() { - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") - local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) - log "SSH into instance with: ssh ${ssh_user}@${public_ip}" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 600 - log "Debug period ended, shutting down" - else - log "Shutting down immediately (debug mode not enabled)" - fi - shutdown -h now - } - - terminate_instance() { - local reason="$1" - local instance_id=$(get_metadata "instance-id") - - echo "========================================" | tee -a /var/log/runner-setup.log - log "FATAL ERROR DETECTED" - log "Reason: $reason" - log "Instance: $instance_id" - log "Script location: $(pwd)" - log "User: $(whoami)" - log "Debug trace available in: /var/log/runner-debug.log" - echo "========================================" | tee -a /var/log/runner-setup.log - - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi - - flush_cloudwatch_logs - debug_sleep_and_shutdown - exit 1 - } - - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - tar -xzf /tmp/runner.tar.gz - - if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then - log "Dependencies exist, skipping install" - else - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >$dn 2>&1 - local deps_result=$? - set -e - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing manually..." - if command -v dnf >$dn 2>&1; then - sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true - elif command -v yum >$dn 2>&1; then - sudo yum install -y libicu >$dn 2>&1 || true - elif command -v apt-get >$dn 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >$dn 2>&1 || true - sudo apt-get install -y libicu-dev >$dn 2>&1 || true - fi - fi - fi - fi - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" - else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } - - EOSF - - chmod +x $B/runner-common.sh - source $B/runner-common.sh - - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM - - REGISTRATION_TIMEOUT="300" - if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 - fi - logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" - touch $V-watchdog-active - ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f $V-registered ]; then - log "Watchdog: Registration marker not found after timeout" - touch $V-watchdog-terminate - kill -TERM $$ 2>/dev/null || true + # Essential variables from template substitution + export debug="" + export homedir="/home/ec2-user" + export repo="omsf-eco-infra/awsinfratesting" + export runner_tokens="test" + export runner_labels="label" + export cloudwatch_logs_group="" + export runner_grace_period="61" + export runner_initial_grace_period="181" + export runner_poll_interval="11" + export runner_registration_timeout="300" + export max_instance_lifetime="360" + export runners_per_instance="1" + export runner_release="test.tar.gz" + export ssh_pubkey="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" + export instance_name="" + export action_sha="abc123def456789012345678901234567890abcd" + + # Custom userdata from user (if any) + export userdata="" + export script="echo 'Hello, World!'" + + # Log prefixes + export log_prefix_job_started="Job started:" + export log_prefix_job_completed="Job completed:" + + # Fetch and execute the main script from GitHub + # action_sha has already been resolved from action_ref in Python for security and consistency + SCRIPT_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/${action_sha}/src/ec2_gha/scripts/runner-setup.sh" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching main script from: $SCRIPT_URL (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log + + # Try to download with retries + for i in {1..5}; do + if curl -sSL "$SCRIPT_URL" -o /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break + elif wget -q "$SCRIPT_URL" -O /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break else - log "Watchdog: Registration marker found, exiting normally" - fi - rm -f $V-watchdog-active - ) & - REGISTRATION_WATCHDOG_PID=$! - log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid - - - - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - - INSTANCE_TYPE=$(get_metadata "instance-type") - INSTANCE_ID=$(get_metadata "instance-id") - REGION=$(get_metadata "placement/region") - AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - - MAX_LIFETIME_MINUTES=360 - log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c " - sleep ${MAX_LIFETIME_MINUTES}m - echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true - shutdown -h now 2>/dev/null || { - halt -f 2>/dev/null || { - if [ -w /proc/sysrq-trigger ]; then - echo 1 > /proc/sys/kernel/sysrq 2>/dev/null - echo o > /proc/sysrq-trigger 2>/dev/null - fi - reboot -f 2>/dev/null || true - } - } - " > /var/log/max-lifetime.log 2>&1 & - - if [ "" != "" ]; then - log "Installing CloudWatch agent" - ( - if command -v dpkg >/dev/null 2>&1; then - log "Detected dpkg-based system" - wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb - elif command -v rpm >/dev/null 2>&1; then - log "Detected rpm-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm - rpm -U ./amazon-cloudwatch-agent.rpm - rm amazon-cloudwatch-agent.rpm - else - log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" - fi - P='"file_path":' - G=',"log_group_name":"","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$homedir" - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {$P"/var/log/runner-setup.log"${G}runner-setup$Z, - {$P"/var/log/runner-debug.log"${G}runner-debug$Z, - {$P"/tmp/job-started-hook.log"${G}job-started$Z, - {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, - {$P"/tmp/termination-check.log"${G}termination$Z, - {$P"/tmp/runner-*-config.log"${G}runner-config$Z, - {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, - {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z - ]}}}} - EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s - log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" - fi - - if [ -n "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" ]; then - log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - mkdir -p "$homedir/.ssh" - chmod 700 "$homedir/.ssh" - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC test@host" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi - log "SSH key added for user $DEFAULT_USER" - fi - - log "Working directory: $homedir" - cd "$homedir" - - echo "echo 'Hello, World!'" > pre-runner-script.sh - log "Running pre-runner script" - source pre-runner-script.sh - export RUNNER_ALLOW_RUNASROOT=1 - - RUNNERS_PER_INSTANCE=1 - - ARCH=$(uname -m) - if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" - else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" - fi - - if command -v curl >/dev/null 2>&1; then - curl -L $RUNNER_URL -o /tmp/runner.tar.gz - elif command -v wget >/dev/null 2>&1; then - wget -q $RUNNER_URL -O /tmp/runner.tar.gz - else - log_error "Neither curl nor wget found. Cannot download runner." - terminate_instance "No download tool available" - fi - log "Downloaded runner binary" - - cat > $B/job-started-hook.sh << EOFS - #!/bin/bash - exec >> /tmp/job-started-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" - mkdir -p $V-jobs - echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity $V-has-run-job - EOFS - - cat > $B/job-completed-hook.sh << EOFC - #!/bin/bash - exec >> /tmp/job-completed-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" - rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity - EOFC - - cat > $B/check-runner-termination.sh << EOFT - #!/bin/bash - exec >> /tmp/termination-check.log 2>&1 - source $B/runner-common.sh - A="$V-last-activity" - J="$V-jobs" - H="$V-has-run-job" - - RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) - if [ \$RUNNER_PROCS -eq 0 ]; then - if ls \$J/*.job 2>/dev/null | grep -q .; then - log "WARNING: Found job files but no runner processes - cleaning up stale jobs" - rm -f \$J/*.job - fi - fi - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - deregister_all_runners - flush_cloudwatch_logs - debug_sleep_and_shutdown - else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" - fi - EOFT - - chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - - mkdir -p $V-jobs - touch $V-last-activity - - cat > /etc/systemd/system/runner-termination-check.service << EOF - [Unit] - Description=Check GitHub runner termination conditions - After=network.target - [Service] - Type=oneshot - Environment="RUNNER_GRACE_PERIOD=61" - Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=$B/check-runner-termination.sh - EOF - - cat > /etc/systemd/system/runner-termination-check.timer << EOF - [Unit] - Description=Periodic GitHub runner termination check - Requires=runner-termination-check.service - [Timer] - OnBootSec=60s - OnUnitActiveSec=11s - [Install] - WantedBy=timers.target - EOF - - systemctl daemon-reload - systemctl enable runner-termination-check.timer - systemctl start runner-termination-check.timer - - METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" - if [ -n "" ]; then - INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') - METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" - fi - - log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - - export -f configure_runner - export -f log - export -f log_error - export -f get_metadata - export -f flush_cloudwatch_logs - export -f deregister_all_runners - export -f debug_sleep_and_shutdown - export -f wait_for_dpkg_lock - - IFS=' ' read -ra tokens <<< "test" - IFS='|' read -ra labels <<< "label" - - num_runners=${#tokens[@]} - log "Configuring $num_runners runner(s) in parallel" - - pids=() - for i in ${!tokens[@]}; do - token=${tokens[$i]} - label=${labels[$i]:-} - if [ -z "$token" ]; then - log_error "No token for runner $i" - continue + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Failed to download script (attempt $i), retrying..." | tee -a /var/log/runner-setup.log + sleep 2 fi - ( - trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR - configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" - echo $? > /tmp/runner-$i-status - ) & - pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" - done - log "Waiting for all runner configurations to complete..." - failed=0 - succeeded=0 - for i in ${!pids[@]}; do - wait ${pids[$i]} - if [ -f /tmp/runner-$i-status ]; then - status=$(cat /tmp/runner-$i-status) - rm -f /tmp/runner-$i-status - if [ "$status" != "0" ]; then - log_error "Runner $i configuration failed" - failed=$((failed + 1)) - else - succeeded=$((succeeded + 1)) - fi + if [ $i -eq 5 ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download runner setup script after 5 attempts" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi done - if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then - terminate_instance "All runners failed to register" - elif [ $failed -gt 0 ]; then - log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." - fi - - if [ $succeeded -gt 0 ]; then - log "$succeeded runner(s) registered and started successfully" - touch $V-registered - else - log_error "No runners registered successfully" - fi - - if [ -f $V-watchdog.pid ]; then - WATCHDOG_PID=$(cat $V-watchdog.pid) - kill $WATCHDOG_PID 2>/dev/null || true - rm -f $V-watchdog.pid + # Verify we got something + if [ ! -s /tmp/runner-setup.sh ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Downloaded script is empty" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi - touch $V-started - chmod o+x $homedir - for RUNNER_DIR in $homedir/runner-*; do - [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" - done - + # Make it executable and run it + chmod +x /tmp/runner-setup.sh + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Executing runner setup script" | tee -a /var/log/runner-setup.log + exec /tmp/runner-setup.sh ''' # --- # name: test_build_user_data_with_cloudwatch @@ -1636,516 +289,67 @@ #!/bin/bash set -e - exec 2> >(tee -a /var/log/runner-debug.log >&2) - - debug="" - - [ "" = "true" ] || [ "" = "True" ] || [ "" = "1" ] && set -x - - homedir="/home/ec2-user" - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - for user in ubuntu ec2-user centos admin debian fedora alpine arch; do - if id "$user" &>/dev/null; then - homedir="/home/$user" - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Auto-detected homedir: $homedir" | tee -a /var/log/runner-setup.log - break - fi - done - - if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then - homedir=$(getent passwd | awk -F: '$3 >= 1000 && $3 < 65534 && $6 ~ /^\\/home\\// {print $6}' | while read dir; do - if [ -d "$dir" ]; then - echo "$dir" - break - fi - done) - if [ -z "$homedir" ]; then - homedir="/home/ec2-user" # Ultimate fallback - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using fallback homedir: $homedir" | tee -a /var/log/runner-setup.log - else - owner=$(stat -c "%U" "$homedir" 2>/dev/null || stat -f "%Su" "$homedir" 2>/dev/null) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Detected homedir: $homedir (owner: $owner)" | tee -a /var/log/runner-setup.log - fi - fi - else - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using specified homedir: $homedir" | tee -a /var/log/runner-setup.log - fi - export homedir - - B=/usr/local/bin - V=/var/run/github-runner - - cat > $B/runner-common.sh << EOSF - homedir="$homedir" - debug="$debug" - export homedir debug - - EOSF - - cat >> $B/runner-common.sh << 'EOSF' - - log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/runner-setup.log; } - log_error() { log "ERROR: $1" >&2; } - - dn=/dev/null - - wait_for_dpkg_lock() { - local t=120 - local L=/var/lib/dpkg/lock - while fuser $L-frontend >$dn 2>&1 || fuser $L >$dn 2>&1; do - if [ $t -le 0 ]; then - log "WARNING: dpkg lock t, proceeding anyway" - break - fi - log "dpkg is locked, waiting... ($t seconds remaining)" - sleep 5 - t=$((t - 5)) - done - } - - flush_cloudwatch_logs() { - log "Stopping CloudWatch agent to flush logs" - if systemctl is-active --quiet amazon-cloudwatch-agent; then - systemctl stop amazon-cloudwatch-agent 2>$dn || /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop -m ec2 2>$dn || true - fi - } - - get_metadata() { - local path="$1" - local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 300" http://169.254.169.254/latest/api/token 2>$dn || true) - if [ -n "$token" ]; then - curl -s -H "X-aws-ec2-metadata-token: $token" "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - else - curl -s "http://169.254.169.254/latest/meta-data/$path" 2>$dn || echo "unknown" - fi - return 0 # Always return success to avoid set -e issues - } - - deregister_all_runners() { - for RUNNER_DIR in $homedir/runner-*; do - if [ -d "$RUNNER_DIR" ] && [ -f "$RUNNER_DIR/config.sh" ]; then - log "Deregistering runner in $RUNNER_DIR" - cd "$RUNNER_DIR" - pkill -INT -f "$RUNNER_DIR/run.sh" 2>$dn || true - sleep 1 - if [ -f "$RUNNER_DIR/.runner-token" ]; then - TOKEN=$(cat "$RUNNER_DIR/.runner-token") - RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token $TOKEN 2>&1 - log "Deregistration exit: $?" - fi - fi - done - } - - debug_sleep_and_shutdown() { - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug: Sleeping 600s before shutdown..." - local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") - local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) - log "SSH into instance with: ssh ${ssh_user}@${public_ip}" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 600 - log "Debug period ended, shutting down" - else - log "Shutting down immediately (debug mode not enabled)" - fi - shutdown -h now - } - - terminate_instance() { - local reason="$1" - local instance_id=$(get_metadata "instance-id") - - echo "========================================" | tee -a /var/log/runner-setup.log - log "FATAL ERROR DETECTED" - log "Reason: $reason" - log "Instance: $instance_id" - log "Script location: $(pwd)" - log "User: $(whoami)" - log "Debug trace available in: /var/log/runner-debug.log" - echo "========================================" | tee -a /var/log/runner-setup.log - - if [ -f "$homedir/config.sh" ] && [ -n "${RUNNER_TOKEN:-}" ]; then - cd "$homedir" && ./config.sh remove --token "${RUNNER_TOKEN}" || true - fi - - flush_cloudwatch_logs - debug_sleep_and_shutdown - exit 1 - } - - configure_runner() { - local idx=$1 - local token=$2 - local labels=$3 - local homedir=$4 - local repo=$5 - local instance_id=$6 - local runner_grace_period=$7 - local runner_initial_grace_period=$8 - - log "Configuring runner $idx..." - - local runner_dir="$homedir/runner-$idx" - mkdir -p "$runner_dir" - cd "$runner_dir" - tar -xzf /tmp/runner.tar.gz - - if [ -f ./bin/installdependencies.sh ]; then - if command -v dpkg >$dn 2>&1 && dpkg -l libicu[0-9]* 2>$dn | grep -q ^ii; then - log "Dependencies exist, skipping install" - else - log "Installing dependencies..." - set +e - sudo ./bin/installdependencies.sh >$dn 2>&1 - local deps_result=$? - set -e - if [ $deps_result -ne 0 ]; then - log "Dependencies script failed, installing manually..." - if command -v dnf >$dn 2>&1; then - sudo dnf install -y libicu lttng-ust >$dn 2>&1 || true - elif command -v yum >$dn 2>&1; then - sudo yum install -y libicu >$dn 2>&1 || true - elif command -v apt-get >$dn 2>&1; then - wait_for_dpkg_lock - sudo apt-get update >$dn 2>&1 || true - sudo apt-get install -y libicu-dev >$dn 2>&1 || true - fi - fi - fi - fi - - echo "$token" > .runner-token - - cat > .env << EOF - ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/job-started-hook.sh - ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/job-completed-hook.sh - RUNNER_HOME=$runner_dir - RUNNER_INDEX=$idx - RUNNER_GRACE_PERIOD=$runner_grace_period - RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period - EOF - - local runner_name="ec2-$instance_id-$idx" - RUNNER_ALLOW_RUNASROOT=1 ./config.sh --url "https://github.com/$repo" --token "$token" --labels "$labels" --name "$runner_name" --disableupdate --unattended 2>&1 | tee /tmp/runner-$idx-config.log - - if grep -q "Runner successfully added" /tmp/runner-$idx-config.log; then - log "Runner $idx registered successfully" + # Essential variables from template substitution + export debug="" + export homedir="/home/ec2-user" + export repo="omsf-eco-infra/awsinfratesting" + export runner_tokens="test" + export runner_labels="label" + export cloudwatch_logs_group="/aws/ec2/github-runners" + export runner_grace_period="61" + export runner_initial_grace_period="181" + export runner_poll_interval="11" + export runner_registration_timeout="300" + export max_instance_lifetime="360" + export runners_per_instance="1" + export runner_release="test.tar.gz" + export ssh_pubkey="" + export instance_name="" + export action_sha="abc123def456789012345678901234567890abcd" + + # Custom userdata from user (if any) + export userdata="" + export script="echo 'Hello, World!'" + + # Log prefixes + export log_prefix_job_started="Job started:" + export log_prefix_job_completed="Job completed:" + + # Fetch and execute the main script from GitHub + # action_sha has already been resolved from action_ref in Python for security and consistency + SCRIPT_URL="https://raw.githubusercontent.com/Open-Athena/ec2-gha/${action_sha}/src/ec2_gha/scripts/runner-setup.sh" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching main script from: $SCRIPT_URL (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log + + # Try to download with retries + for i in {1..5}; do + if curl -sSL "$SCRIPT_URL" -o /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break + elif wget -q "$SCRIPT_URL" -O /tmp/runner-setup.sh; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Successfully downloaded runner setup script (attempt $i)" | tee -a /var/log/runner-setup.log + break else - log_error "Failed to register runner $idx" - return 1 - fi - - RUNNER_ALLOW_RUNASROOT=1 nohup ./run.sh > $dn 2>&1 & - local pid=$! - log "Started runner $idx in $runner_dir (PID: $pid)" - - return 0 - } - - EOSF - - chmod +x $B/runner-common.sh - source $B/runner-common.sh - - logger "EC2-GHA: Starting userdata script" - trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR - trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR - trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM - - REGISTRATION_TIMEOUT="300" - if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then - logger "EC2-GHA: Invalid timeout '$REGISTRATION_TIMEOUT', using default 300" - REGISTRATION_TIMEOUT=300 - fi - logger "EC2-GHA: Registration timeout set to $REGISTRATION_TIMEOUT seconds" - touch $V-watchdog-active - ( - log "Watchdog: Starting $REGISTRATION_TIMEOUT second timeout" - sleep $REGISTRATION_TIMEOUT - if [ ! -f $V-registered ]; then - log "Watchdog: Registration marker not found after timeout" - touch $V-watchdog-terminate - kill -TERM $$ 2>/dev/null || true - else - log "Watchdog: Registration marker found, exiting normally" - fi - rm -f $V-watchdog-active - ) & - REGISTRATION_WATCHDOG_PID=$! - log "Watchdog PID: $REGISTRATION_WATCHDOG_PID" - echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid - - - - exec >> /var/log/runner-setup.log 2>&1 - log "Starting runner setup" - - INSTANCE_TYPE=$(get_metadata "instance-type") - INSTANCE_ID=$(get_metadata "instance-id") - REGION=$(get_metadata "placement/region") - AZ=$(get_metadata "placement/availability-zone") - log "Instance metadata: Type=${INSTANCE_TYPE} ID=${INSTANCE_ID} Region=${REGION} AZ=${AZ}" - - MAX_LIFETIME_MINUTES=360 - log "Setting up maximum lifetime timeout: ${MAX_LIFETIME_MINUTES} minutes" - nohup bash -c " - sleep ${MAX_LIFETIME_MINUTES}m - echo '[$(date)] Maximum lifetime reached' 2>/dev/null || true - shutdown -h now 2>/dev/null || { - halt -f 2>/dev/null || { - if [ -w /proc/sysrq-trigger ]; then - echo 1 > /proc/sys/kernel/sysrq 2>/dev/null - echo o > /proc/sysrq-trigger 2>/dev/null - fi - reboot -f 2>/dev/null || true - } - } - " > /var/log/max-lifetime.log 2>&1 & - - if [ "/aws/ec2/github-runners" != "" ]; then - log "Installing CloudWatch agent" - ( - if command -v dpkg >/dev/null 2>&1; then - log "Detected dpkg-based system" - wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb - elif command -v rpm >/dev/null 2>&1; then - log "Detected rpm-based system" - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm - rpm -U ./amazon-cloudwatch-agent.rpm - rm amazon-cloudwatch-agent.rpm - else - log "WARNING: Unable to detect package manager, skipping CloudWatch agent installation" - fi - P='"file_path":' - G=',"log_group_name":"/aws/ec2/github-runners","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$homedir" - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF - {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ - {$P"/var/log/runner-setup.log"${G}runner-setup$Z, - {$P"/var/log/runner-debug.log"${G}runner-debug$Z, - {$P"/tmp/job-started-hook.log"${G}job-started$Z, - {$P"/tmp/job-completed-hook.log"${G}job-completed$Z, - {$P"/tmp/termination-check.log"${G}termination$Z, - {$P"/tmp/runner-*-config.log"${G}runner-config$Z, - {$P"$H/_diag/Runner_**.log"${G}runner-diag$Z, - {$P"$H/_diag/Worker_**.log"${G}worker-diag$Z - ]}}}} - EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s - log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" - fi - - if [ -n "" ]; then - log "Configuring SSH access" - DEFAULT_USER=$(stat -c "%U" "$homedir" 2>/dev/null || echo "root") - mkdir -p "$homedir/.ssh" - chmod 700 "$homedir/.ssh" - echo "" >> "$homedir/.ssh/authorized_keys" - chmod 600 "$homedir/.ssh/authorized_keys" - if [ "$DEFAULT_USER" != "root" ]; then - chown -R "$DEFAULT_USER:$DEFAULT_USER" "$homedir/.ssh" - fi - log "SSH key added for user $DEFAULT_USER" - fi - - log "Working directory: $homedir" - cd "$homedir" - - echo "echo 'Hello, World!'" > pre-runner-script.sh - log "Running pre-runner script" - source pre-runner-script.sh - export RUNNER_ALLOW_RUNASROOT=1 - - RUNNERS_PER_INSTANCE=1 - - ARCH=$(uname -m) - if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - RUNNER_URL=$(echo "test.tar.gz" | sed 's/x64/arm64/g') - log "ARM detected, using: $RUNNER_URL" - else - RUNNER_URL="test.tar.gz" - log "x64 detected, using: $RUNNER_URL" - fi - - if command -v curl >/dev/null 2>&1; then - curl -L $RUNNER_URL -o /tmp/runner.tar.gz - elif command -v wget >/dev/null 2>&1; then - wget -q $RUNNER_URL -O /tmp/runner.tar.gz - else - log_error "Neither curl nor wget found. Cannot download runner." - terminate_instance "No download tool available" - fi - log "Downloaded runner binary" - - cat > $B/job-started-hook.sh << EOFS - #!/bin/bash - exec >> /tmp/job-started-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job started: \${GITHUB_JOB}" - mkdir -p $V-jobs - echo '{"status":"running","runner":"'\$I'"}' > $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity $V-has-run-job - EOFS - - cat > $B/job-completed-hook.sh << EOFC - #!/bin/bash - exec >> /tmp/job-completed-hook.log 2>&1 - I="\${RUNNER_INDEX:-0}" - echo "[\$(date)] Runner-\$I: Job completed: \${GITHUB_JOB}" - rm -f $V-jobs/\${GITHUB_RUN_ID}-\${GITHUB_JOB}-\$I.job - touch $V-last-activity - EOFC - - cat > $B/check-runner-termination.sh << EOFT - #!/bin/bash - exec >> /tmp/termination-check.log 2>&1 - source $B/runner-common.sh - A="$V-last-activity" - J="$V-jobs" - H="$V-has-run-job" - - RUNNER_PROCS=\$(pgrep -f "Runner.Listener" | wc -l) - if [ \$RUNNER_PROCS -eq 0 ]; then - if ls \$J/*.job 2>/dev/null | grep -q .; then - log "WARNING: Found job files but no runner processes - cleaning up stale jobs" - rm -f \$J/*.job + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Failed to download script (attempt $i), retrying..." | tee -a /var/log/runner-setup.log + sleep 2 fi - fi - - [ ! -f "\$A" ] && touch "\$A" - L=\$(stat -c %Y "\$A" 2>/dev/null || echo 0) - N=\$(date +%s) - I=\$((N-L)) - [ -f "\$H" ] && G=\${RUNNER_GRACE_PERIOD:-60} || G=\${RUNNER_INITIAL_GRACE_PERIOD:-180} - R=\$(grep -l '"status":"running"' \$J/*.job 2>/dev/null | wc -l || echo 0) - if [ \$R -eq 0 ] && [ \$I -gt \$G ]; then - log "TERMINATING: idle \$I > grace \$G" - deregister_all_runners - flush_cloudwatch_logs - debug_sleep_and_shutdown - else - [ \$R -gt 0 ] && log "\$R job(s) running" || log "Idle \$I/\$G sec" - fi - EOFT - - chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh - - mkdir -p $V-jobs - touch $V-last-activity - - cat > /etc/systemd/system/runner-termination-check.service << EOF - [Unit] - Description=Check GitHub runner termination conditions - After=network.target - [Service] - Type=oneshot - Environment="RUNNER_GRACE_PERIOD=61" - Environment="RUNNER_INITIAL_GRACE_PERIOD=181" - ExecStart=$B/check-runner-termination.sh - EOF - - cat > /etc/systemd/system/runner-termination-check.timer << EOF - [Unit] - Description=Periodic GitHub runner termination check - Requires=runner-termination-check.service - [Timer] - OnBootSec=60s - OnUnitActiveSec=11s - [Install] - WantedBy=timers.target - EOF - - systemctl daemon-reload - systemctl enable runner-termination-check.timer - systemctl start runner-termination-check.timer - - METADATA_LABELS=",${INSTANCE_ID},${INSTANCE_TYPE}" - if [ -n "" ]; then - INSTANCE_NAME_LABEL=$(echo "" | tr ' /' '-' | tr -cd '[:alnum:]-_#') - METADATA_LABELS="${METADATA_LABELS},${INSTANCE_NAME_LABEL}" - fi - - log "Setting up $RUNNERS_PER_INSTANCE runner(s)" - - export -f configure_runner - export -f log - export -f log_error - export -f get_metadata - export -f flush_cloudwatch_logs - export -f deregister_all_runners - export -f debug_sleep_and_shutdown - export -f wait_for_dpkg_lock - - IFS=' ' read -ra tokens <<< "test" - IFS='|' read -ra labels <<< "label" - num_runners=${#tokens[@]} - log "Configuring $num_runners runner(s) in parallel" - - pids=() - for i in ${!tokens[@]}; do - token=${tokens[$i]} - label=${labels[$i]:-} - if [ -z "$token" ]; then - log_error "No token for runner $i" - continue + if [ $i -eq 5 ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download runner setup script after 5 attempts" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi - ( - trap 'echo "Subshell error on line $LINENO" >&2; exit 1' ERR - configure_runner $i "$token" "${label}$METADATA_LABELS" "$homedir" "omsf-eco-infra/awsinfratesting" "$INSTANCE_ID" "61" "181" - echo $? > /tmp/runner-$i-status - ) & - pids+=($!) - log "Started configuration for runner $i (PID: ${pids[-1]})" done - log "Waiting for all runner configurations to complete..." - failed=0 - succeeded=0 - for i in ${!pids[@]}; do - wait ${pids[$i]} - if [ -f /tmp/runner-$i-status ]; then - status=$(cat /tmp/runner-$i-status) - rm -f /tmp/runner-$i-status - if [ "$status" != "0" ]; then - log_error "Runner $i configuration failed" - failed=$((failed + 1)) - else - succeeded=$((succeeded + 1)) - fi - fi - done - - if [ $succeeded -eq 0 ] && [ $failed -gt 0 ]; then - terminate_instance "All runners failed to register" - elif [ $failed -gt 0 ]; then - log "WARNING: $failed runner(s) failed, but $succeeded succeeded. Continuing with partial capacity." - fi - - if [ $succeeded -gt 0 ]; then - log "$succeeded runner(s) registered and started successfully" - touch $V-registered - else - log_error "No runners registered successfully" - fi - - if [ -f $V-watchdog.pid ]; then - WATCHDOG_PID=$(cat $V-watchdog.pid) - kill $WATCHDOG_PID 2>/dev/null || true - rm -f $V-watchdog.pid + # Verify we got something + if [ ! -s /tmp/runner-setup.sh ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Downloaded script is empty" | tee -a /var/log/runner-setup.log + shutdown -h now + exit 1 fi - touch $V-started - chmod o+x $homedir - for RUNNER_DIR in $homedir/runner-*; do - [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" - done - + # Make it executable and run it + chmod +x /tmp/runner-setup.sh + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Executing runner setup script" | tee -a /var/log/runner-setup.log + exec /tmp/runner-setup.sh ''' # --- diff --git a/tests/test_start.py b/tests/test_start.py index e370de4..3fba731 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -24,15 +24,38 @@ def base_aws_params(): @pytest.fixture(scope="function") -def aws(base_aws_params): +def aws(base_aws_params, monkeypatch): with mock_aws(): - yield StartAWS(**base_aws_params) + monkeypatch.setenv("INPUT_ACTION_REF", "v2") + # Mock subprocess.run to handle both git config and git rev-parse + def mock_subprocess_run(cmd, *args, **kwargs): + if cmd[0] == 'git' and cmd[1] == 'config': + # git config command - just return success + mock_result = Mock() + mock_result.returncode = 0 + mock_result.stdout = "" + mock_result.stderr = "" + return mock_result + elif cmd[0] == 'git' and cmd[1] == 'rev-parse': + # git rev-parse command - return mock SHA + mock_result = Mock() + mock_result.returncode = 0 + mock_result.stdout = "abc123def456789012345678901234567890abcd\n" + mock_result.stderr = "" + return mock_result + else: + raise ValueError(f"Unexpected subprocess call: {cmd}") + + with patch("ec2_gha.start.subprocess.run", side_effect=mock_subprocess_run): + yield StartAWS(**base_aws_params) @pytest.fixture(scope="function") def aws_params_user_data(): """User data params for AWS params tests""" return { + "action_ref": "v2", # Test ref + "action_sha": "abc123def456789012345678901234567890abcd", # Mock SHA for testing "cloudwatch_logs_group": "", # Empty = disabled "debug": "", # Empty = disabled "github_run_id": "16725250800", @@ -152,7 +175,7 @@ def test_build_aws_params(complete_params, aws_params_user_data, github_env, sna assert params == snapshot -def test_auto_home_dir(complete_params): +def test_auto_home_dir(complete_params, monkeypatch): """Test that home_dir is set to AUTO when not provided""" params = complete_params.copy() params['home_dir'] = "" @@ -160,7 +183,26 @@ def test_auto_home_dir(complete_params): aws.gh_runner_tokens = ["test-token"] aws.runner_release = "https://example.com/runner.tar.gz" - with patch("boto3.client") as mock_client: + monkeypatch.setenv("INPUT_ACTION_REF", "v2") + + # Mock subprocess.run for git commands + def mock_subprocess_run(cmd, *args, **kwargs): + if cmd[0] == 'git' and cmd[1] == 'config': + mock_result = Mock() + mock_result.returncode = 0 + return mock_result + elif cmd[0] == 'git' and cmd[1] == 'rev-parse': + mock_result = Mock() + mock_result.returncode = 0 + mock_result.stdout = "abc123def456789012345678901234567890abcd\n" + return mock_result + else: + raise ValueError(f"Unexpected subprocess call: {cmd}") + + with ( + patch("boto3.client") as mock_client, + patch("ec2_gha.start.subprocess.run", side_effect=mock_subprocess_run) + ): mock_ec2 = Mock() mock_client.return_value = mock_ec2 @@ -318,17 +360,34 @@ def test_create_instances_missing_release(aws): aws.create_instances() -def test_create_instances_sets_auto_home_dir(base_aws_params): +def test_create_instances_sets_auto_home_dir(base_aws_params, monkeypatch): """Test that home_dir is set to AUTO when not provided""" params = base_aws_params.copy() params['home_dir'] = "" + monkeypatch.setenv("INPUT_ACTION_REF", "v2") + + # Mock subprocess.run for git commands + def mock_subprocess_run(cmd, *args, **kwargs): + if cmd[0] == 'git' and cmd[1] == 'config': + mock_result = Mock() + mock_result.returncode = 0 + return mock_result + elif cmd[0] == 'git' and cmd[1] == 'rev-parse': + mock_result = Mock() + mock_result.returncode = 0 + mock_result.stdout = "abc123def456789012345678901234567890abcd\n" + return mock_result + else: + raise ValueError(f"Unexpected subprocess call: {cmd}") + with mock_aws(): aws = StartAWS(**params) aws.gh_runner_tokens = ["test-token"] aws.runner_release = "https://example.com/runner.tar.gz" - with patch("boto3.client") as mock_client: + with patch("boto3.client") as mock_client, \ + patch("ec2_gha.start.subprocess.run", side_effect=mock_subprocess_run): mock_ec2 = Mock() mock_client.return_value = mock_ec2 From cc64d49f4be0d879959db02a1358413779b9da21 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 15:48:36 -0400 Subject: [PATCH 59/69] `Debug mode: false=off, true/trace=set -x only, number=set -x + sleep N minutes before shutdown` --- .github/workflows/runner.yml | 4 ++-- README.md | 1 + action.yml | 2 +- src/ec2_gha/scripts/runner-setup.sh | 7 +++++-- src/ec2_gha/templates/shared-functions.sh | 12 +++++++++--- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index c7d45c1..eecc2b4 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -39,9 +39,9 @@ on: required: false type: string debug: - description: "Enable debug output (set -x) in runner setup script" + description: "Debug mode: false=off, true/trace=set -x only, number=set -x + sleep N minutes before shutdown" required: false - type: boolean + type: string default: false ec2_home_dir: description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR, then auto-detection)" diff --git a/README.md b/README.md index ed5b193..5240618 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): - `ec2_key_name` - EC2 key pair name (for [SSH access]) - `instance_count` - Number of instances to create (default: 1, for parallel jobs) - `instance_name` - Name tag template for EC2 instances. Uses Python string.Template format with variables: `$repo`, `$name` (workflow filename stem), `$workflow` (full workflow name), `$ref`, `$run` (number), `$idx` (0-based instance index for multi-instance launches). Default: `$repo/$name#$run` (or `$repo/$name#$run $idx` for multi-instance) +- `debug` - Debug mode: `false`=off, `true`/`trace`=set -x only, number=set -x + sleep N minutes before shutdown (for troubleshooting) - `ec2_root_device_size` - Root device size in GB (default: 0 = use AMI default) - `ec2_security_group_id` - Security group ID (required for [SSH access], should expose inbound port 22) - `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing) diff --git a/action.yml b/action.yml index 438bfc1..9dbe16e 100644 --- a/action.yml +++ b/action.yml @@ -21,7 +21,7 @@ inputs: description: "CloudWatch Logs group name for streaming runner logs (leave empty to disable)" required: false debug: - description: "Enable debug output (set -x) in runner setup script" + description: "Debug mode: false=off, true/trace=set -x only, number=set -x + sleep N minutes before shutdown" required: false ec2_home_dir: description: "Home directory on the AWS instance (falls back to vars.EC2_HOME_DIR, then auto-detection)" diff --git a/src/ec2_gha/scripts/runner-setup.sh b/src/ec2_gha/scripts/runner-setup.sh index 35427c0..a0440d8 100755 --- a/src/ec2_gha/scripts/runner-setup.sh +++ b/src/ec2_gha/scripts/runner-setup.sh @@ -7,8 +7,11 @@ set -e # Enable debug tracing to a file for troubleshooting exec 2> >(tee -a /var/log/runner-debug.log >&2) -# Conditionally enable debug mode -[ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ] && set -x +# Conditionally enable debug mode (set -x) for tracing +# Debug can be: true/True/trace (trace only), or a number (trace + sleep minutes) +if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "trace" ] || [[ "$debug" =~ ^[0-9]+$ ]]; then + set -x +fi # Determine home directory early since it's needed by shared functions if [ -z "$homedir" ] || [ "$homedir" = "AUTO" ]; then diff --git a/src/ec2_gha/templates/shared-functions.sh b/src/ec2_gha/templates/shared-functions.sh index 7309686..e6f75db 100644 --- a/src/ec2_gha/templates/shared-functions.sh +++ b/src/ec2_gha/templates/shared-functions.sh @@ -62,15 +62,21 @@ deregister_all_runners() { # Function to handle debug mode sleep and shutdown debug_sleep_and_shutdown() { - if [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "1" ]; then - log "Debug: Sleeping 600s before shutdown..." + # Check if debug is a number (sleep duration in minutes) + if [[ "$debug" =~ ^[0-9]+$ ]]; then + local sleep_minutes="$debug" + local sleep_seconds=$((sleep_minutes * 60)) + log "Debug: Sleeping ${sleep_minutes} minutes before shutdown..." # Detect the SSH user from the home directory local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) log "SSH into instance with: ssh ${ssh_user}@${public_ip}" log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" - sleep 600 + sleep "$sleep_seconds" log "Debug period ended, shutting down" + elif [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "trace" ]; then + # Just tracing enabled, no sleep + log "Shutting down immediately (debug tracing enabled but no sleep requested)" else log "Shutting down immediately (debug mode not enabled)" fi From d04bbe761e650ab24499d57507fb540fab6dbea8 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 15:54:49 -0400 Subject: [PATCH 60/69] `ec2_root_device_size: Root disk size in GB (0=AMI default, +N=AMI+N GB for testing, e.g. +2)` --- .github/workflows/README.md | 19 ++++++++++++++ .github/workflows/runner.yml | 2 +- README.md | 3 ++- action.yml | 2 +- src/ec2_gha/__main__.py | 2 +- src/ec2_gha/start.py | 24 ++++++++++++++---- tests/test_start.py | 49 ++++++++++++++++++++++++++++++++++-- 7 files changed, 90 insertions(+), 11 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1aa8e18..69b408e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -14,6 +14,8 @@ For documentation about the main workflow, [`runner.yml`](runner.yml), see [the - [`instances-mtx` – multiple instances for parallel jobs](#instances-mtx) - [`runners-mtx` – multiple runners on single instance](#runners-mtx) - [`jobs-split` – different job types on separate instances](#jobs-split) +- [Stress testing](#stress-tests) + - [`test-disk-full` – disk-full scenario testing](#test-disk-full) - [Real-world example: Mamba installation testing](#mamba) @@ -72,6 +74,23 @@ Useful regression test, demonstrates and verifies features. - **Instance type:** `t3.medium` - **Use case:** Pipeline with dedicated instances per stage +## Stress testing + +### [`test-disk-full`](test-disk-full.yml) – disk-full scenario testing +- Tests runner behavior when disk space is exhausted +- **Configurable parameters:** + - `disk_size`: Root disk size (`0`=AMI default, `+N`=AMI+N GB, e.g., `+2`) + - `fill_strategy`: How to fill disk (`gradual`, `immediate`, or `during-tests`) + - `debug`: Debug mode (`false`, `true`/`trace`, or number for trace+sleep) + - `max_instance_lifetime`: Maximum lifetime before forced shutdown (default: 15 minutes) +- **Features tested:** + - Heartbeat mechanism for detecting stuck jobs + - Stale job file detection and cleanup + - Worker/Listener process monitoring + - Robust shutdown with multiple fallback methods +- **Instance type:** `t3.medium` (default) +- **Use case:** Verifying robustness in resource-constrained environments + ## Real-world example: [Mamba installation testing](https://github.com/Open-Athena/mamba/blob/gha/.github/workflows/install.yaml) - Tests different versions of `mamba_ssm` package on GPU instances - **Customizes `instance_name`**: `"$repo/$name==${{ inputs.mamba_version }} (#$run)"` diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index eecc2b4..2ee6d7f 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -68,7 +68,7 @@ on: required: false type: string ec2_root_device_size: - description: "Root device size in GB (0 = use AMI default)" + description: "Root disk size in GB (0=AMI default, +N=AMI+N GB for testing, e.g. +2)" required: false type: string default: "0" diff --git a/README.md b/README.md index 5240618..a15ec0b 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ Example workflows demonstrating ec2-gha capabilities are in [`.github/workflows/ ### Test Suite - [`demos.yml`](.github/workflows/demos.yml) - Runs all demos for regression testing +- [`test-disk-full.yml`](.github/workflows/test-disk-full.yml) - Stress test for disk-full scenarios with configurable fill strategies See [`.github/workflows/README.md`](.github/workflows/README.md) for detailed descriptions of each demo. @@ -118,7 +119,7 @@ Many of these fall back to corresponding `vars.*` (if not provided as `inputs`): - `instance_count` - Number of instances to create (default: 1, for parallel jobs) - `instance_name` - Name tag template for EC2 instances. Uses Python string.Template format with variables: `$repo`, `$name` (workflow filename stem), `$workflow` (full workflow name), `$ref`, `$run` (number), `$idx` (0-based instance index for multi-instance launches). Default: `$repo/$name#$run` (or `$repo/$name#$run $idx` for multi-instance) - `debug` - Debug mode: `false`=off, `true`/`trace`=set -x only, number=set -x + sleep N minutes before shutdown (for troubleshooting) -- `ec2_root_device_size` - Root device size in GB (default: 0 = use AMI default) +- `ec2_root_device_size` - Root disk size in GB: `0`=AMI default, `+N`=AMI+N GB for testing (e.g., `+2` for AMI size + 2GB), or explicit size in GB - `ec2_security_group_id` - Security group ID (required for [SSH access], should expose inbound port 22) - `max_instance_lifetime` - Maximum instance lifetime in minutes before automatic shutdown (falls back to `vars.MAX_INSTANCE_LIFETIME`, default: 360 = 6 hours; generally should not be relevant, instances shut down within 1-2mins of jobs completing) - `runner_grace_period` - Grace period in seconds before terminating after last job completes (default: 60) diff --git a/action.yml b/action.yml index 9dbe16e..053277d 100644 --- a/action.yml +++ b/action.yml @@ -39,7 +39,7 @@ inputs: description: "Name of an EC2 key pair to use for SSH access (falls back to vars.EC2_KEY_NAME)" required: false ec2_root_device_size: - description: "Root device size in GB (0 = use AMI default)" + description: "Root disk size in GB (0=AMI default, +N=AMI+N GB for testing, e.g. +2)" required: false ec2_security_group_id: description: "AWS security group ID (falls back to vars.EC2_SECURITY_GROUP_ID)" diff --git a/src/ec2_gha/__main__.py b/src/ec2_gha/__main__.py index f0b5b18..90a2655 100644 --- a/src/ec2_gha/__main__.py +++ b/src/ec2_gha/__main__.py @@ -39,7 +39,7 @@ def main(): .update_state("INPUT_EC2_INSTANCE_PROFILE", "iam_instance_profile") .update_state("INPUT_EC2_INSTANCE_TYPE", "instance_type") .update_state("INPUT_EC2_KEY_NAME", "key_name") - .update_state("INPUT_EC2_ROOT_DEVICE_SIZE", "root_device_size", type_hint=int) + .update_state("INPUT_EC2_ROOT_DEVICE_SIZE", "root_device_size", type_hint=str) .update_state("INPUT_EC2_SECURITY_GROUP_ID", "security_group_id") .update_state("INPUT_EC2_USERDATA", "userdata") .update_state("INPUT_EXTRA_GH_LABELS", "labels") diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 6799273..6774b45 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -95,7 +95,7 @@ class StartAWS(CreateCloudInstance): A comma-separated list of labels to apply to the runner. Defaults to an empty string. max_instance_lifetime : str Maximum instance lifetime in minutes before automatic shutdown. Defaults to "360" (6 hours). - root_device_size : int + root_device_size : str The size of the root device. Defaults to 0 which uses the default. runner_initial_grace_period : str Grace period in seconds before terminating if no jobs have started. Defaults to "180". @@ -134,7 +134,7 @@ class StartAWS(CreateCloudInstance): key_name: str = "" labels: str = "" max_instance_lifetime: str = "360" - root_device_size: int = 0 + root_device_size: str = "0" runner_grace_period: str = "60" runner_initial_grace_period: str = "180" runner_poll_interval: str = "10" @@ -351,9 +351,23 @@ def _modify_root_disk_size(self, client, params: dict) -> dict: block_devices = deepcopy(image_options["Images"][0]["BlockDeviceMappings"]) for idx, block_device in enumerate(block_devices): if block_device["DeviceName"] == root_device_name: - if self.root_device_size > 0: - block_devices[idx]["Ebs"]["VolumeSize"] = self.root_device_size + size_str = self.root_device_size.strip() + if size_str.startswith('+'): + # +N means "AMI size + N GB" + # Useful for disk-full testing: +2 means AMI size + 2GB + current_size = block_device.get("Ebs", {}).get("VolumeSize", 8) + buffer_gb = int(size_str[1:]) + new_size = current_size + buffer_gb + block_devices[idx]["Ebs"]["VolumeSize"] = new_size params["BlockDeviceMappings"] = block_devices + print(f"Setting disk size to {new_size}GB (AMI default {current_size}GB + {buffer_gb}GB)") + elif size_str != "0": + # Explicit size in GB + new_size = int(size_str) + if new_size > 0: + block_devices[idx]["Ebs"]["VolumeSize"] = new_size + params["BlockDeviceMappings"] = block_devices + # else: size_str == "0" means use AMI default, do nothing break else: raise e @@ -451,7 +465,7 @@ def create_instances(self) -> dict[str, str]: "userdata": self.userdata, } params = self._build_aws_params(user_data_params, idx=idx) - if self.root_device_size > 0: + if self.root_device_size != "0": params = self._modify_root_disk_size(ec2, params) # Check UserData size before calling AWS diff --git a/tests/test_start.py b/tests/test_start.py index 3fba731..931738c 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -119,7 +119,7 @@ def complete_params(base_aws_params): "gh_runner_tokens": ["test"], "iam_instance_profile": "test", "labels": "", - "root_device_size": 100, + "root_device_size": "100", "runner_release": "test.tar.gz", "security_group_id": "test", "subnet_id": "test", @@ -253,6 +253,7 @@ def mock_describe_images(**kwargs): error_response={"Error": {"Code": "DryRunOperation"}}, operation_name="DescribeImages" ) + # This is the second call without DryRun return mock_image_data mock_client.describe_images = mock_describe_images @@ -300,9 +301,53 @@ def test_modify_root_disk_size_permission_error(complete_params): assert 'AccessDenied' in str(exc_info.value) +def test_modify_root_disk_size_plus_syntax(complete_params): + """Test the +N syntax for adding GB to AMI default size""" + mock_client = Mock() + complete_params["root_device_size"] = "+5" + + # Mock image data with default size of 8GB + mock_image_data = { + "Images": [{ + "RootDeviceName": "/dev/sda1", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": True, + "VolumeSize": 8, + "VolumeType": "gp3" + } + } + ] + }] + } + + def mock_describe_images(**kwargs): + if kwargs.get('DryRun', False): + raise ClientError( + error_response={"Error": {"Code": "DryRunOperation"}}, + operation_name="DescribeImages" + ) + return mock_image_data + + mock_client.describe_images = mock_describe_images + aws = StartAWS(**complete_params) + + # Test with +5 (should be 8 + 5 = 13) + result = aws._modify_root_disk_size(mock_client, {}) + assert result["BlockDeviceMappings"][0]["Ebs"]["VolumeSize"] == 13 + + # Test with +2 + complete_params["root_device_size"] = "+2" + aws = StartAWS(**complete_params) + result = aws._modify_root_disk_size(mock_client, {}) + assert result["BlockDeviceMappings"][0]["Ebs"]["VolumeSize"] == 10 + + def test_modify_root_disk_size_no_change(complete_params): mock_client = Mock() - complete_params["root_device_size"] = 0 + complete_params["root_device_size"] = "0" mock_image_data = { "Images": [{ From 0792e03271768bd0035f44b682784ccf6b8c98f5 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 15:57:31 -0400 Subject: [PATCH 61/69] "heartbeat" job files, detect stale jobs / out-of-disk, add `test-disk-full.yml` example --- .github/workflows/test-disk-full.yml | 208 +++++++++++++++++- CLAUDE.md | 13 +- README.md | 6 + .../scripts/check-runner-termination.sh | 47 ++++ 4 files changed, 265 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-disk-full.yml b/.github/workflows/test-disk-full.yml index 1c0b0ab..c7ec32f 100644 --- a/.github/workflows/test-disk-full.yml +++ b/.github/workflows/test-disk-full.yml @@ -1,8 +1,210 @@ name: Test Disk Full + on: workflow_dispatch: + inputs: + disk_size: + description: 'Root disk size in GB (0=AMI default, +N=AMI+N GB, e.g. +2 for AMI+2GB)' + required: false + type: string + default: '+2' # 2GB more than the AMI size + fill_strategy: + description: 'How to fill disk: gradual, immediate, or during-tests' + required: false + type: choice + options: + - gradual + - immediate + - during-tests + default: gradual + debug: + description: 'Debug mode: false=off, true/trace=trace only, number=trace+sleep N minutes' + required: false + type: string + default: 'true' + instance_type: + description: 'Instance type' + required: false + type: string + default: 't3.medium' + max_instance_lifetime: + description: 'Max instance lifetime in minutes (default: 15)' + required: false + type: string + default: '15' + +permissions: + id-token: write + contents: read + jobs: - placeholder: - runs-on: ubuntu-latest + launch: + name: Launch runner + uses: ./.github/workflows/runner.yml + secrets: + GH_SA_TOKEN: ${{ secrets.GH_SA_TOKEN }} + with: + ec2_image_id: ami-0ca5a2f40c2601df6 # Ubuntu 24.04 x86_64 in us-east-1 + ec2_instance_type: ${{ inputs.instance_type }} + ec2_root_device_size: ${{ inputs.disk_size }} + debug: ${{ inputs.debug }} + max_instance_lifetime: ${{ inputs.max_instance_lifetime }} + + test-disk-full: + name: Fill disk (${{ inputs.fill_strategy }}) + needs: launch + runs-on: ${{ needs.launch.outputs.id }} + steps: - - run: echo "Placeholder workflow to preserve workflow name in GitHub Actions UI" + - name: Check initial disk usage + run: | + echo "=== Initial disk usage ===" + df -h / + echo "" + echo "=== Largest directories ===" + du -sh /* 2>/dev/null | sort -hr | head -10 || true + + - name: Fill disk immediately + if: inputs.fill_strategy == 'immediate' + run: | + echo "=== Filling disk immediately ===" + # Create a large file that leaves only ~100MB free + AVAILABLE=$(df / | awk 'NR==2 {print int($4/1024)-100}') + if [ $AVAILABLE -gt 0 ]; then + echo "Creating ${AVAILABLE}MB file to fill disk..." + dd if=/dev/zero of=/tmp/disk_filler bs=1M count=$AVAILABLE 2>/dev/null || true + fi + echo "=== Disk usage after fill ===" + df -h / + + - name: Fill disk gradually + if: inputs.fill_strategy == 'gradual' + run: | + echo "=== Filling disk gradually ===" + COUNTER=0 + while true; do + AVAILABLE=$(df / | awk 'NR==2 {print int($4/1024)}') + if [ $AVAILABLE -lt 500 ]; then + echo "Disk nearly full (${AVAILABLE}MB remaining), creating final files..." + # Fill remaining space with smaller files + for i in {1..10}; do + dd if=/dev/zero of=/tmp/gradual_fill_${COUNTER}_${i} bs=1M count=50 2>/dev/null || break + done + break + fi + echo "Creating 500MB file (${AVAILABLE}MB currently available)..." + dd if=/dev/zero of=/tmp/gradual_fill_${COUNTER} bs=1M count=500 2>/dev/null || break + COUNTER=$((COUNTER + 1)) + df -h / + sleep 2 + done + echo "=== Final disk usage ===" + df -h / + + - name: Setup Python project for test + if: inputs.fill_strategy == 'during-tests' + run: | + echo "=== Setting up Python project that will fill disk during tests ===" + cat > setup.py << 'EOF' + from setuptools import setup, find_packages + + setup( + name="disk-filler-test", + version="0.1.0", + packages=find_packages(), + python_requires=">=3.8", + install_requires=[ + "pytest>=7.0.0", + "numpy>=1.20.0", # Large package + "pandas>=1.3.0", # Large package + "scipy>=1.7.0", # Large package + "matplotlib>=3.4.0", # Large package + "scikit-learn>=1.0.0", # Large package + "torch>=2.0.0", # Very large package + "transformers>=4.30.0", # Very large package + ], + ) + EOF + + mkdir -p tests + cat > tests/test_disk_filler.py << 'EOF' + import os + import tempfile + import pytest + + def test_create_large_arrays(): + """Create large arrays to consume memory and disk (via swap/tmp)""" + import numpy as np + arrays = [] + for i in range(10): + # Create 100MB arrays + arr = np.random.random((1024, 1024, 100)) + arrays.append(arr) + # Also write to temp file + with tempfile.NamedTemporaryFile(delete=False, dir='/tmp') as f: + np.save(f, arr) + print(f"Created array {i+1}/10") + + def test_generate_files(): + """Generate many temporary files""" + for i in range(100): + with tempfile.NamedTemporaryFile(delete=False, dir='/tmp', + prefix=f'test_file_{i}_') as f: + # Write 10MB to each file + f.write(os.urandom(10 * 1024 * 1024)) + if i % 10 == 0: + print(f"Generated {i+1}/100 files") + + def test_disk_space_check(): + """Check if we're out of disk space""" + import shutil + usage = shutil.disk_usage('/') + percent_used = (usage.used / usage.total) * 100 + print(f"Disk usage: {percent_used:.1f}%") + print(f"Free space: {usage.free / (1024**3):.2f} GB") + # This test "passes" even when disk is full to see behavior + assert percent_used > 0 + EOF + + echo "=== Installing packages (this will consume disk space) ===" + pip install -e . || true + + echo "=== Running tests that fill disk ===" + pytest tests/ -v || true + + echo "=== Final disk usage ===" + df -h / + + - name: Try to write when disk is full + if: always() + run: | + echo "=== Testing write operations with full disk ===" + # Try various write operations to see what fails + echo "Test" > /tmp/test_write.txt 2>&1 || echo "Failed to write to /tmp" + echo "Test" > ~/test_write.txt 2>&1 || echo "Failed to write to home" + touch /tmp/test_touch 2>&1 || echo "Failed to touch file" + mkdir /tmp/test_mkdir 2>&1 || echo "Failed to create directory" + + # Check if we can still run commands + echo "=== Can we still run basic commands? ===" + date || echo "date command failed" + pwd || echo "pwd command failed" + whoami || echo "whoami command failed" + + - name: Monitor termination behavior + if: always() + run: | + echo "=== Monitoring termination behavior ===" + echo "This job will complete soon. Watch the runner logs to see if:" + echo "1. The termination check detects disk full state" + echo "2. The instance can successfully shut down" + echo "3. The robust shutdown methods are triggered" + echo "" + echo "Current disk usage:" + df -h / + echo "" + echo "Checking runner processes:" + ps aux | grep -E '[R]unner|[c]heck-runner-termination' || true + echo "" + echo "Last entries in termination check log:" + tail -20 /tmp/termination-check.log 2>/dev/null || echo "No termination check log found" diff --git a/CLAUDE.md b/CLAUDE.md index 566adf6..ebb6a88 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,18 +112,19 @@ The runner uses a polling-based approach to determine when to terminate: 1. **Job Tracking**: GitHub runner hooks track job lifecycle - `job-started-hook.sh`: Creates JSON files in `/var/run/github-runner-jobs/` - `job-completed-hook.sh`: Removes job files and updates activity timestamp + - Heartbeat mechanism: Active jobs touch their files periodically 2. **Periodic Polling**: Systemd timer runs `check-runner-termination.sh` every `runner_poll_interval` seconds (default: 10s) - - Checks for running jobs (files with `"status":"running"`) - - If no running jobs, compares idle time against grace period + - Checks for running jobs by verifying both Runner.Listener AND Runner.Worker processes + - Detects stale job files (older than 3× poll interval, likely disk full) + - Handles Worker process death (job completed but hook couldn't run) - Grace periods: - `runner_initial_grace_period` (default: 180s) - Before first job - `runner_grace_period` (default: 60s) - Between jobs -3. **Effective Termination Time**: Due to polling interval, actual termination occurs: - - **Minimum**: grace_period seconds after last activity - - **Maximum**: grace_period + runner_poll_interval seconds - - Example: With 60s grace and 10s poll → terminates 60-70s after last job +3. **Robustness Features**: + - **Process Monitoring**: Distinguishes between idle Listener and active Worker + - **Hook Script Separation**: Scripts fetched from GitHub for maintainability 4. **Clean Shutdown Sequence**: - Stop runner processes gracefully (SIGINT with timeout) diff --git a/README.md b/README.md index a15ec0b..ea84557 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,8 @@ The runner uses [GitHub Actions runner hooks][hooks] to track job lifecycle and #### Job Tracking - **Start/End Hooks**: Creates/removes JSON files in `/var/run/github-runner-jobs/` when jobs start/end +- **Heartbeat Mechanism**: Active jobs update their file timestamps periodically to detect stuck jobs +- **Process Monitoring**: Checks both Runner.Listener and Runner.Worker processes to verify jobs are truly running - **Activity Tracking**: Updates `/var/run/github-runner-last-activity` timestamp on job events #### Termination Conditions @@ -219,6 +221,10 @@ The systemd timer checks every `runner_poll_interval` seconds (default: 10s) and - `runner_initial_grace_period` (default: 180s) - Before first job - `runner_grace_period` (default: 60s) - Between jobs +#### Robustness Features +- **Stale Job Detection**: Removes job files older than 3× poll interval (likely disk full) +- **Worker Process Detection**: Distinguishes between idle runners and active jobs + #### Clean Shutdown Sequence 1. Stop runner processes gracefully (SIGINT) 2. Deregister runners from GitHub diff --git a/src/ec2_gha/scripts/check-runner-termination.sh b/src/ec2_gha/scripts/check-runner-termination.sh index 36ddf55..d73d165 100644 --- a/src/ec2_gha/scripts/check-runner-termination.sh +++ b/src/ec2_gha/scripts/check-runner-termination.sh @@ -25,6 +25,53 @@ if [ $RUNNER_PROCS -eq 0 ]; then fi fi +# Check job files and update timestamps for active runners +# This creates a heartbeat mechanism to detect stuck/failed job completion +for job_file in $J/*.job; do + [ -f "$job_file" ] || continue + if grep -q '"status":"running"' "$job_file" 2>/dev/null; then + # Extract runner number from job file name (format: RUNID-JOBNAME-RUNNER.job) + runner_num=$(basename "$job_file" .job | rev | cut -d- -f1 | rev) + + # For a job to be truly running, we need BOTH Listener AND Worker processes + # Listener alone means the runner is idle/waiting, not actually running a job + listener_alive=$(pgrep -f "runner-${runner_num}/.*Runner.Listener" 2>/dev/null | wc -l) + worker_alive=$(pgrep -f "runner-${runner_num}/.*Runner.Worker" 2>/dev/null | wc -l) + + if [ $listener_alive -gt 0 ] && [ $worker_alive -gt 0 ]; then + # Both processes exist, job is truly running - update heartbeat + touch "$job_file" 2>/dev/null || true + elif [ $listener_alive -gt 0 ] && [ $worker_alive -eq 0 ]; then + # Listener exists but no Worker - job has likely failed/completed but hook couldn't run + job_age=$((N - $(stat -c %Y "$job_file" 2>/dev/null || echo 0))) + log "WARNING: Runner $runner_num Listener alive but Worker dead - job likely completed (file age: ${job_age}s)" + rm -f "$job_file" + touch "$A" # Update last activity since we just cleaned up a job + else + # No Listener at all - runner is completely dead + job_age=$((N - $(stat -c %Y "$job_file" 2>/dev/null || echo 0))) + log "WARNING: Job file $(basename $job_file) exists but runner $runner_num is dead (file age: ${job_age}s)" + rm -f "$job_file" + fi + fi +done + +# Now check for stale job files that couldn't be touched (e.g., disk full) +# With polling every ${RUNNER_POLL_INTERVAL:-10}s, files should never be older than ~30s +# If they are, something is preventing the touch (likely disk full) +STALE_THRESHOLD=$((${RUNNER_POLL_INTERVAL:-10} * 3)) # 3x the poll interval +for job_file in $J/*.job; do + [ -f "$job_file" ] || continue + if grep -q '"status":"running"' "$job_file" 2>/dev/null; then + job_age=$((N - $(stat -c %Y "$job_file" 2>/dev/null || echo 0))) + if [ $job_age -gt $STALE_THRESHOLD ]; then + log "ERROR: Job file $(basename $job_file) is stale (${job_age}s old, threshold ${STALE_THRESHOLD}s)" + log "Touch must be failing (disk full?) - removing stale job file" + rm -f "$job_file" + fi + fi +done + # Ensure activity file exists and get its timestamp [ ! -f "$A" ] && touch "$A" L=$(stat -c %Y "$A" 2>/dev/null || echo 0) From 1fe02b0142b1accddb9562a0adc921e6590ed2a7 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 18 Sep 2025 15:57:06 -0400 Subject: [PATCH 62/69] more robust `debug_sleep_and_shutdown` --- CLAUDE.md | 2 ++ README.md | 1 + src/ec2_gha/templates/shared-functions.sh | 27 +++++++++++++++++------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ebb6a88..c1afd32 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,12 +124,14 @@ The runner uses a polling-based approach to determine when to terminate: 3. **Robustness Features**: - **Process Monitoring**: Distinguishes between idle Listener and active Worker + - **Fallback Termination**: Multiple shutdown methods with increasing force - **Hook Script Separation**: Scripts fetched from GitHub for maintainability 4. **Clean Shutdown Sequence**: - Stop runner processes gracefully (SIGINT with timeout) - Deregister all runners from GitHub - Flush CloudWatch logs (if configured) + - Execute shutdown with fallbacks (`systemctl poweroff`, `shutdown -h now`, `halt -f`) ### AWS Resource Tagging By default, launched EC2 instances are Tagged with: diff --git a/README.md b/README.md index ea84557..d5714a7 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,7 @@ The systemd timer checks every `runner_poll_interval` seconds (default: 10s) and #### Robustness Features - **Stale Job Detection**: Removes job files older than 3× poll interval (likely disk full) - **Worker Process Detection**: Distinguishes between idle runners and active jobs +- **Multiple Shutdown Methods**: Uses robust termination with fallback to `shutdown -h now` #### Clean Shutdown Sequence 1. Stop runner processes gracefully (SIGINT) diff --git a/src/ec2_gha/templates/shared-functions.sh b/src/ec2_gha/templates/shared-functions.sh index e6f75db..b7167fa 100644 --- a/src/ec2_gha/templates/shared-functions.sh +++ b/src/ec2_gha/templates/shared-functions.sh @@ -66,21 +66,34 @@ debug_sleep_and_shutdown() { if [[ "$debug" =~ ^[0-9]+$ ]]; then local sleep_minutes="$debug" local sleep_seconds=$((sleep_minutes * 60)) - log "Debug: Sleeping ${sleep_minutes} minutes before shutdown..." + log "Debug: Sleeping ${sleep_minutes} minutes before shutdown..." || true # Detect the SSH user from the home directory local ssh_user=$(basename "$homedir" 2>$dn || echo "ec2-user") local public_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) - log "SSH into instance with: ssh ${ssh_user}@${public_ip}" - log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" + log "SSH into instance with: ssh ${ssh_user}@${public_ip}" || true + log "Then check: /var/log/runner-setup.log and /var/log/runner-debug.log" || true sleep "$sleep_seconds" - log "Debug period ended, shutting down" + log "Debug period ended, shutting down" || true elif [ "$debug" = "true" ] || [ "$debug" = "True" ] || [ "$debug" = "trace" ]; then # Just tracing enabled, no sleep - log "Shutting down immediately (debug tracing enabled but no sleep requested)" + log "Shutting down immediately (debug tracing enabled but no sleep requested)" || true else - log "Shutting down immediately (debug mode not enabled)" + log "Shutting down immediately (debug mode not enabled)" || true fi - shutdown -h now + + # Try multiple shutdown methods as fallbacks (important when disk is full) + shutdown -h now 2>/dev/null || { + # If shutdown fails, try halt + halt -f 2>/dev/null || { + # If halt fails, try sysrq if available (Linux only) + if [ -w /proc/sysrq-trigger ]; then + echo 1 > /proc/sys/kernel/sysrq 2>/dev/null + echo o > /proc/sysrq-trigger 2>/dev/null + fi + # Last resort: force immediate reboot + reboot -f 2>/dev/null || true + } + } } # Function to handle fatal errors and terminate the instance From 74253021ad98fe6b64cf8d6a434172fa3c3baf05 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 24 Sep 2025 12:24:17 -0400 Subject: [PATCH 63/69] `terminate_instance` when no runners registered successfully --- src/ec2_gha/scripts/runner-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ec2_gha/scripts/runner-setup.sh b/src/ec2_gha/scripts/runner-setup.sh index a0440d8..c5dee7d 100755 --- a/src/ec2_gha/scripts/runner-setup.sh +++ b/src/ec2_gha/scripts/runner-setup.sh @@ -367,6 +367,7 @@ if [ $succeeded -gt 0 ]; then touch $V-registered else log_error "No runners registered successfully" + terminate_instance "No runners registered successfully" fi # Kill registration watchdog now that runners are registered From d4c1714908b668c7860e1e2e9e248fb9c533daa0 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 24 Sep 2025 12:35:31 -0400 Subject: [PATCH 64/69] CW init fix, fail on CW init failure --- src/ec2_gha/scripts/runner-setup.sh | 59 +++++++++++++++-------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/ec2_gha/scripts/runner-setup.sh b/src/ec2_gha/scripts/runner-setup.sh index c5dee7d..b0a342f 100755 --- a/src/ec2_gha/scripts/runner-setup.sh +++ b/src/ec2_gha/scripts/runner-setup.sh @@ -140,37 +140,40 @@ nohup bash -c " # Configure CloudWatch Logs if a log group is specified if [ "$cloudwatch_logs_group" != "" ]; then log "Installing CloudWatch agent" - ( - if command -v dpkg >/dev/null 2>&1; then - wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb - dpkg -i -E ./amazon-cloudwatch-agent.deb - rm amazon-cloudwatch-agent.deb - elif command -v rpm >/dev/null 2>&1; then - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm - rpm -U ./amazon-cloudwatch-agent.rpm - rm amazon-cloudwatch-agent.rpm - fi - # Build CloudWatch config with factored strings - P='"file_path":' - G=',"log_group_name":"'$cloudwatch_logs_group'","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$homedir" - cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF + if command -v dpkg >/dev/null 2>&1; then + wait_for_dpkg_lock + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + dpkg -i -E ./amazon-cloudwatch-agent.deb + rm amazon-cloudwatch-agent.deb + elif command -v rpm >/dev/null 2>&1; then + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + rpm -U ./amazon-cloudwatch-agent.rpm + rm amazon-cloudwatch-agent.rpm + fi + + # Build CloudWatch config with factored strings + G=',"log_group_name":"'$cloudwatch_logs_group'","log_stream_name":"{instance_id}/' + Z='","timezone":"UTC"}' + H="$homedir" + cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF {"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ -{"$P/var/log/runner-setup.log${G}runner-setup$Z", -{"$P/var/log/runner-debug.log${G}runner-debug$Z", -{"$P/tmp/job-started-hook.log${G}job-started$Z", -{"$P/tmp/job-completed-hook.log${G}job-completed$Z", -{"$P/tmp/termination-check.log${G}termination$Z", -{"$P/tmp/runner-*-config.log${G}runner-config$Z", -{"$P$H/_diag/Runner_**.log${G}runner-diag$Z", -{"$P$H/_diag/Worker_**.log${G}worker-diag$Z" +{"file_path":"/var/log/runner-setup.log${G}runner-setup$Z"}, +{"file_path":"/var/log/runner-debug.log${G}runner-debug$Z"}, +{"file_path":"/tmp/job-started-hook.log${G}job-started$Z"}, +{"file_path":"/tmp/job-completed-hook.log${G}job-completed$Z"}, +{"file_path":"/tmp/termination-check.log${G}termination$Z"}, +{"file_path":"/tmp/runner-*-config.log${G}runner-config$Z"}, +{"file_path":"$H/_diag/Runner_**.log${G}runner-diag$Z"}, +{"file_path":"$H/_diag/Worker_**.log${G}worker-diag$Z"} ]}}}} EOF - /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s - log "CloudWatch agent started" - ) || log "WARNING: CloudWatch agent installation failed, continuing without it" + + if ! /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s; then + log_error "Failed to start CloudWatch agent" + terminate_instance "CloudWatch agent startup failed" + fi + + log "CloudWatch agent started successfully" fi # Configure SSH access if public key provided (useful for debugging) From bb72f09ec38962eef6c7197ed11ea1b588071fba Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 24 Sep 2025 12:38:40 -0400 Subject: [PATCH 65/69] reformat / de-obfuscate some vars --- .../scripts/check-runner-termination.sh | 6 +- src/ec2_gha/scripts/job-completed-hook.sh | 7 +- src/ec2_gha/scripts/job-started-hook.sh | 9 +- src/ec2_gha/scripts/runner-setup.sh | 88 +++++++++++-------- 4 files changed, 63 insertions(+), 47 deletions(-) diff --git a/src/ec2_gha/scripts/check-runner-termination.sh b/src/ec2_gha/scripts/check-runner-termination.sh index d73d165..3388d24 100644 --- a/src/ec2_gha/scripts/check-runner-termination.sh +++ b/src/ec2_gha/scripts/check-runner-termination.sh @@ -8,9 +8,9 @@ exec >> /tmp/termination-check.log 2>&1 source /usr/local/bin/runner-common.sh # File paths for tracking -A="/var/run/github-runner-last-activity" -J="/var/run/github-runner-jobs" -H="/var/run/github-runner-has-run-job" +A="$RUNNER_STATE_DIR/last-activity" +J="$RUNNER_STATE_DIR/jobs" +H="$RUNNER_STATE_DIR/has-run-job" # Current timestamp N=$(date +%s) diff --git a/src/ec2_gha/scripts/job-completed-hook.sh b/src/ec2_gha/scripts/job-completed-hook.sh index f0a83f6..4bb214d 100644 --- a/src/ec2_gha/scripts/job-completed-hook.sh +++ b/src/ec2_gha/scripts/job-completed-hook.sh @@ -5,6 +5,9 @@ exec >> /tmp/job-completed-hook.log 2>&1 +# Source common variables +source /usr/local/bin/runner-common.sh + # Get runner index from environment (defaults to 0 for single-runner instances) I="${RUNNER_INDEX:-0}" @@ -13,7 +16,7 @@ I="${RUNNER_INDEX:-0}" echo "[$(date)] Runner-$I: LOG_PREFIX_JOB_COMPLETED ${GITHUB_JOB}" # Remove the job tracking file to indicate this runner no longer has an active job -rm -f /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$I.job +rm -f $RUNNER_STATE_DIR/jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$I.job # Update activity timestamp to reset the idle timer -touch /var/run/github-runner-last-activity +touch $RUNNER_STATE_DIR/last-activity diff --git a/src/ec2_gha/scripts/job-started-hook.sh b/src/ec2_gha/scripts/job-started-hook.sh index 55315d6..e80591c 100644 --- a/src/ec2_gha/scripts/job-started-hook.sh +++ b/src/ec2_gha/scripts/job-started-hook.sh @@ -5,6 +5,9 @@ exec >> /tmp/job-started-hook.log 2>&1 +# Source common variables +source /usr/local/bin/runner-common.sh + # Get runner index from environment (defaults to 0 for single-runner instances) I="${RUNNER_INDEX:-0}" @@ -14,8 +17,8 @@ echo "[$(date)] Runner-$I: LOG_PREFIX_JOB_STARTED Runner-$I: ${GITHUB_JOB}" # Create a job tracking file to indicate this runner has an active job # Format: RUNID-JOBNAME-RUNNER.job -mkdir -p /var/run/github-runner-jobs -echo '{"status":"running","runner":"'$I'"}' > /var/run/github-runner-jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$I.job +mkdir -p $RUNNER_STATE_DIR/jobs +echo '{"status":"running","runner":"'$I'"}' > $RUNNER_STATE_DIR/jobs/${GITHUB_RUN_ID}-${GITHUB_JOB}-$I.job # Update activity timestamps to reset the idle timer -touch /var/run/github-runner-last-activity /var/run/github-runner-has-run-job +touch $RUNNER_STATE_DIR/last-activity $RUNNER_STATE_DIR/has-run-job diff --git a/src/ec2_gha/scripts/runner-setup.sh b/src/ec2_gha/scripts/runner-setup.sh index b0a342f..2174dfd 100755 --- a/src/ec2_gha/scripts/runner-setup.sh +++ b/src/ec2_gha/scripts/runner-setup.sh @@ -46,8 +46,9 @@ fi export homedir # Set common paths -B=/usr/local/bin -V=/var/run/github-runner +BIN_DIR=/usr/local/bin +RUNNER_STATE_DIR=/var/run/github-runner +mkdir -p $RUNNER_STATE_DIR # Fetch shared functions from GitHub echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching shared functions from GitHub (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log @@ -59,26 +60,27 @@ if ! curl -sSL "$FUNCTIONS_URL" -o /tmp/shared-functions.sh && ! wget -q "$FUNCT fi # Write shared functions that will be used by multiple scripts -cat > $B/runner-common.sh << EOSF +cat > $BIN_DIR/runner-common.sh << EOSF # Auto-generated shared functions and variables # Set homedir for scripts that source this file homedir="$homedir" debug="$debug" -export homedir debug +RUNNER_STATE_DIR="$RUNNER_STATE_DIR" +export homedir debug RUNNER_STATE_DIR EOSF # Append the downloaded shared functions -cat /tmp/shared-functions.sh >> $B/runner-common.sh +cat /tmp/shared-functions.sh >> $BIN_DIR/runner-common.sh -chmod +x $B/runner-common.sh -source $B/runner-common.sh +chmod +x $BIN_DIR/runner-common.sh +source $BIN_DIR/runner-common.sh logger "EC2-GHA: Starting userdata script" trap 'logger "EC2-GHA: Script failed at line $LINENO with exit code $?"' ERR trap 'terminate_instance "Setup script failed with error on line $LINENO"' ERR # Handle watchdog termination signal -trap 'if [ -f $V-watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM +trap 'if [ -f $RUNNER_STATE_DIR/watchdog-terminate ]; then terminate_instance "No runners registered within timeout"; else terminate_instance "Script terminated"; fi' TERM # Set up registration timeout failsafe - terminate if runner doesn't register in time REGISTRATION_TIMEOUT="$runner_registration_timeout" @@ -86,17 +88,17 @@ if ! [[ "$REGISTRATION_TIMEOUT" =~ ^[0-9]+$ ]]; then REGISTRATION_TIMEOUT=300 fi # Create a marker file for watchdog termination request -touch $V-watchdog-active +touch $RUNNER_STATE_DIR/watchdog-active ( sleep $REGISTRATION_TIMEOUT - if [ ! -f $V-registered ]; then - touch $V-watchdog-terminate + if [ ! -f $RUNNER_STATE_DIR/registered ]; then + touch $RUNNER_STATE_DIR/watchdog-terminate kill -TERM $$ 2>/dev/null || true fi - rm -f $V-watchdog-active + rm -f $RUNNER_STATE_DIR/watchdog-active ) & REGISTRATION_WATCHDOG_PID=$! -echo $REGISTRATION_WATCHDOG_PID > $V-watchdog.pid +echo $REGISTRATION_WATCHDOG_PID > $RUNNER_STATE_DIR/watchdog.pid # Run any custom user data script provided by the user if [ -n "$userdata" ]; then @@ -151,21 +153,29 @@ if [ "$cloudwatch_logs_group" != "" ]; then rm amazon-cloudwatch-agent.rpm fi - # Build CloudWatch config with factored strings - G=',"log_group_name":"'$cloudwatch_logs_group'","log_stream_name":"{instance_id}/' - Z='","timezone":"UTC"}' - H="$homedir" + # Build CloudWatch config cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << EOF -{"agent":{"run_as_user":"cwagent"},"logs":{"logs_collected":{"files":{"collect_list":[ -{"file_path":"/var/log/runner-setup.log${G}runner-setup$Z"}, -{"file_path":"/var/log/runner-debug.log${G}runner-debug$Z"}, -{"file_path":"/tmp/job-started-hook.log${G}job-started$Z"}, -{"file_path":"/tmp/job-completed-hook.log${G}job-completed$Z"}, -{"file_path":"/tmp/termination-check.log${G}termination$Z"}, -{"file_path":"/tmp/runner-*-config.log${G}runner-config$Z"}, -{"file_path":"$H/_diag/Runner_**.log${G}runner-diag$Z"}, -{"file_path":"$H/_diag/Worker_**.log${G}worker-diag$Z"} -]}}}} +{ + "agent": { + "run_as_user": "cwagent" + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { "file_path": "/var/log/runner-setup.log" , "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-setup" , "timezone": "UTC" }, + { "file_path": "/var/log/runner-debug.log" , "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-debug" , "timezone": "UTC" }, + { "file_path": "/tmp/job-started-hook.log" , "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-started" , "timezone": "UTC" }, + { "file_path": "/tmp/job-completed-hook.log" , "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/job-completed", "timezone": "UTC" }, + { "file_path": "/tmp/termination-check.log" , "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/termination" , "timezone": "UTC" }, + { "file_path": "/tmp/runner-*-config.log" , "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-config", "timezone": "UTC" }, + { "file_path": "$homedir/_diag/Runner_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/runner-diag" , "timezone": "UTC" }, + { "file_path": "$homedir/_diag/Worker_**.log", "log_group_name": "$cloudwatch_logs_group", "log_stream_name": "{instance_id}/worker-diag" , "timezone": "UTC" } + ] + } + } + } +} EOF if ! /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s; then @@ -229,7 +239,7 @@ log "Downloaded runner binary" fetch_script() { local script_name="$1" local url="${BASE_URL}/${script_name}" - local dest="${B}/${script_name}" + local dest="${BIN_DIR}/${script_name}" if command -v curl >/dev/null 2>&1; then curl -fsSL "$url" -o "$dest" || { @@ -257,14 +267,14 @@ fetch_script "job-completed-hook.sh" fetch_script "check-runner-termination.sh" # Replace log prefix placeholders with actual values -sed -i "s/LOG_PREFIX_JOB_STARTED/${log_prefix_job_started}/g" $B/job-started-hook.sh -sed -i "s/LOG_PREFIX_JOB_COMPLETED/${log_prefix_job_completed}/g" $B/job-completed-hook.sh +sed -i "s/LOG_PREFIX_JOB_STARTED/${log_prefix_job_started}/g" $BIN_DIR/job-started-hook.sh +sed -i "s/LOG_PREFIX_JOB_COMPLETED/${log_prefix_job_completed}/g" $BIN_DIR/job-completed-hook.sh -chmod +x $B/job-started-hook.sh $B/job-completed-hook.sh $B/check-runner-termination.sh +chmod +x $BIN_DIR/job-started-hook.sh $BIN_DIR/job-completed-hook.sh $BIN_DIR/check-runner-termination.sh # Set up job tracking directory -mkdir -p $V-jobs -touch $V-last-activity +mkdir -p $RUNNER_STATE_DIR/jobs +touch $RUNNER_STATE_DIR/last-activity # Set up periodic termination check using systemd cat > /etc/systemd/system/runner-termination-check.service << EOF @@ -276,7 +286,7 @@ Type=oneshot Environment="RUNNER_GRACE_PERIOD=$runner_grace_period" Environment="RUNNER_INITIAL_GRACE_PERIOD=$runner_initial_grace_period" Environment="RUNNER_POLL_INTERVAL=$runner_poll_interval" -ExecStart=$B/check-runner-termination.sh +ExecStart=$BIN_DIR/check-runner-termination.sh EOF cat > /etc/systemd/system/runner-termination-check.timer << EOF @@ -367,21 +377,21 @@ fi if [ $succeeded -gt 0 ]; then log "$succeeded runner(s) registered and started successfully" - touch $V-registered + touch $RUNNER_STATE_DIR/registered else log_error "No runners registered successfully" terminate_instance "No runners registered successfully" fi # Kill registration watchdog now that runners are registered -if [ -f $V-watchdog.pid ]; then - WATCHDOG_PID=$(cat $V-watchdog.pid) +if [ -f $RUNNER_STATE_DIR/watchdog.pid ]; then + WATCHDOG_PID=$(cat $RUNNER_STATE_DIR/watchdog.pid) kill $WATCHDOG_PID 2>/dev/null || true - rm -f $V-watchdog.pid + rm -f $RUNNER_STATE_DIR/watchdog.pid fi # Final setup - ensure runner directories are accessible for debugging -touch $V-started +touch $RUNNER_STATE_DIR/started chmod o+x $homedir for RUNNER_DIR in $homedir/runner-*; do [ -d "$RUNNER_DIR/_diag" ] && chmod 755 "$RUNNER_DIR/_diag" From 0dde623dce01235d47a2befa67324a9037cefcb7 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Wed, 24 Sep 2025 14:30:05 -0400 Subject: [PATCH 66/69] CW amd fix --- src/ec2_gha/scripts/runner-setup.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ec2_gha/scripts/runner-setup.sh b/src/ec2_gha/scripts/runner-setup.sh index 2174dfd..26d14a1 100755 --- a/src/ec2_gha/scripts/runner-setup.sh +++ b/src/ec2_gha/scripts/runner-setup.sh @@ -142,13 +142,23 @@ nohup bash -c " # Configure CloudWatch Logs if a log group is specified if [ "$cloudwatch_logs_group" != "" ]; then log "Installing CloudWatch agent" + + # Detect architecture for CloudWatch agent + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + CW_ARCH="arm64" + else + CW_ARCH="amd64" + fi + if command -v dpkg >/dev/null 2>&1; then wait_for_dpkg_lock - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/${CW_ARCH}/latest/amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb rm amazon-cloudwatch-agent.deb elif command -v rpm >/dev/null 2>&1; then - wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + # Note: For RPM-based systems, the path structure might differ + wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/${CW_ARCH}/latest/amazon-cloudwatch-agent.rpm rpm -U ./amazon-cloudwatch-agent.rpm rm amazon-cloudwatch-agent.rpm fi From 33a2a68a9b8f62968297a8d98b1a15474c6ca532 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 22 Jun 2026 11:12:00 -0400 Subject: [PATCH 67/69] Pull `python:3.12` base from `public.ecr.aws` mirror The Docker action (`Dockerfile`) is built on the GitHub-hosted launcher before the EC2 runner is created. Pulling `python:3.12` from Docker Hub is subject to anonymous pull rate limits, which can fail the build. `public.ecr.aws/docker/library/python:3.12` is AWS's public mirror of the same official image, with no auth and no anonymous pull limits, so this fixes the rate-limit failures with zero config for callers. Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 420bc11..41ce11a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12 +FROM public.ecr.aws/docker/library/python:3.12 ENV PYTHONUNBUFFERED=1 RUN mkdir app From 7fd84b6a04e3ef56523f212f34e14a782b321ce8 Mon Sep 17 00:00:00 2001 From: Jesse Rusak Date: Thu, 9 Jul 2026 09:52:27 -0400 Subject: [PATCH 68/69] Update gha-runner dependency from upstream --- pyproject.toml | 4 +- src/ec2_gha/start.py | 2 +- uv.lock | 194 ++----------------------------------------- 3 files changed, 8 insertions(+), 192 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7281398..9740999 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ name = "ec2_gha" version = "1.0.0" description = "Start an AWS GitHub Actions Runner" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.12" authors = [{ name = "Ethan Holz", email = "ethan.holz@omsf.io" }] -dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@v1"] +dependencies = ["boto3", "gha_runner @ git+https://github.com/omsf/gha-runner.git@8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6"] [project.optional-dependencies] test = ["pytest", "pytest-cov", "moto[ec2]", "responses", "ruff", "syrupy"] diff --git a/src/ec2_gha/start.py b/src/ec2_gha/start.py index 6774b45..b52f5fe 100644 --- a/src/ec2_gha/start.py +++ b/src/ec2_gha/start.py @@ -309,7 +309,7 @@ def _build_user_data(self, **kwargs) -> str: try: parsed = Template(template_content) - runner_script = parsed.substitute(**kwargs) + runner_script = parsed.substitute(**kwargs).rstrip() # Log the final size for informational purposes script_size = len(runner_script) diff --git a/uv.lock b/uv.lock index 8851867..314e993 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.10" +requires-python = ">=3.12" [[package]] name = "boto3" @@ -48,30 +48,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191, upload-time = "2024-09-04T20:43:30.027Z" }, - { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592, upload-time = "2024-09-04T20:43:32.108Z" }, - { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024, upload-time = "2024-09-04T20:43:34.186Z" }, - { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188, upload-time = "2024-09-04T20:43:36.286Z" }, - { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571, upload-time = "2024-09-04T20:43:38.586Z" }, - { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687, upload-time = "2024-09-04T20:43:40.084Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211, upload-time = "2024-09-04T20:43:41.526Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325, upload-time = "2024-09-04T20:43:43.117Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784, upload-time = "2024-09-04T20:43:45.256Z" }, - { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564, upload-time = "2024-09-04T20:43:46.779Z" }, - { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804, upload-time = "2024-09-04T20:43:48.186Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299, upload-time = "2024-09-04T20:43:49.812Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, @@ -102,28 +78,6 @@ version = "3.4.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, - { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, - { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, - { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, - { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, - { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, - { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, - { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, - { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, - { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, - { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, - { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, - { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, - { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, - { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, - { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, - { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, @@ -175,27 +129,6 @@ version = "7.10.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/14/70/025b179c993f019105b79575ac6edb5e084fb0f0e63f15cdebef4e454fb5/coverage-7.10.6.tar.gz", hash = "sha256:f644a3ae5933a552a29dbb9aa2f90c677a875f80ebea028e5a52a4f429044b90", size = 823736, upload-time = "2025-08-29T15:35:16.668Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/1d/2e64b43d978b5bd184e0756a41415597dfef30fcbd90b747474bd749d45f/coverage-7.10.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70e7bfbd57126b5554aa482691145f798d7df77489a177a6bef80de78860a356", size = 217025, upload-time = "2025-08-29T15:32:57.169Z" }, - { url = "https://files.pythonhosted.org/packages/23/62/b1e0f513417c02cc10ef735c3ee5186df55f190f70498b3702d516aad06f/coverage-7.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e41be6f0f19da64af13403e52f2dec38bbc2937af54df8ecef10850ff8d35301", size = 217419, upload-time = "2025-08-29T15:32:59.908Z" }, - { url = "https://files.pythonhosted.org/packages/e7/16/b800640b7a43e7c538429e4d7223e0a94fd72453a1a048f70bf766f12e96/coverage-7.10.6-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c61fc91ab80b23f5fddbee342d19662f3d3328173229caded831aa0bd7595460", size = 244180, upload-time = "2025-08-29T15:33:01.608Z" }, - { url = "https://files.pythonhosted.org/packages/fb/6f/5e03631c3305cad187eaf76af0b559fff88af9a0b0c180d006fb02413d7a/coverage-7.10.6-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10356fdd33a7cc06e8051413140bbdc6f972137508a3572e3f59f805cd2832fd", size = 245992, upload-time = "2025-08-29T15:33:03.239Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a1/f30ea0fb400b080730125b490771ec62b3375789f90af0bb68bfb8a921d7/coverage-7.10.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80b1695cf7c5ebe7b44bf2521221b9bb8cdf69b1f24231149a7e3eb1ae5fa2fb", size = 247851, upload-time = "2025-08-29T15:33:04.603Z" }, - { url = "https://files.pythonhosted.org/packages/02/8e/cfa8fee8e8ef9a6bb76c7bef039f3302f44e615d2194161a21d3d83ac2e9/coverage-7.10.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2e4c33e6378b9d52d3454bd08847a8651f4ed23ddbb4a0520227bd346382bbc6", size = 245891, upload-time = "2025-08-29T15:33:06.176Z" }, - { url = "https://files.pythonhosted.org/packages/93/a9/51be09b75c55c4f6c16d8d73a6a1d46ad764acca0eab48fa2ffaef5958fe/coverage-7.10.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c8a3ec16e34ef980a46f60dc6ad86ec60f763c3f2fa0db6d261e6e754f72e945", size = 243909, upload-time = "2025-08-29T15:33:07.74Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a6/ba188b376529ce36483b2d585ca7bdac64aacbe5aa10da5978029a9c94db/coverage-7.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7d79dabc0a56f5af990cc6da9ad1e40766e82773c075f09cc571e2076fef882e", size = 244786, upload-time = "2025-08-29T15:33:08.965Z" }, - { url = "https://files.pythonhosted.org/packages/d0/4c/37ed872374a21813e0d3215256180c9a382c3f5ced6f2e5da0102fc2fd3e/coverage-7.10.6-cp310-cp310-win32.whl", hash = "sha256:86b9b59f2b16e981906e9d6383eb6446d5b46c278460ae2c36487667717eccf1", size = 219521, upload-time = "2025-08-29T15:33:10.599Z" }, - { url = "https://files.pythonhosted.org/packages/8e/36/9311352fdc551dec5b973b61f4e453227ce482985a9368305880af4f85dd/coverage-7.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:e132b9152749bd33534e5bd8565c7576f135f157b4029b975e15ee184325f528", size = 220417, upload-time = "2025-08-29T15:33:11.907Z" }, - { url = "https://files.pythonhosted.org/packages/d4/16/2bea27e212c4980753d6d563a0803c150edeaaddb0771a50d2afc410a261/coverage-7.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c706db3cabb7ceef779de68270150665e710b46d56372455cd741184f3868d8f", size = 217129, upload-time = "2025-08-29T15:33:13.575Z" }, - { url = "https://files.pythonhosted.org/packages/2a/51/e7159e068831ab37e31aac0969d47b8c5ee25b7d307b51e310ec34869315/coverage-7.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e0c38dc289e0508ef68ec95834cb5d2e96fdbe792eaccaa1bccac3966bbadcc", size = 217532, upload-time = "2025-08-29T15:33:14.872Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c0/246ccbea53d6099325d25cd208df94ea435cd55f0db38099dd721efc7a1f/coverage-7.10.6-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:752a3005a1ded28f2f3a6e8787e24f28d6abe176ca64677bcd8d53d6fe2ec08a", size = 247931, upload-time = "2025-08-29T15:33:16.142Z" }, - { url = "https://files.pythonhosted.org/packages/7d/fb/7435ef8ab9b2594a6e3f58505cc30e98ae8b33265d844007737946c59389/coverage-7.10.6-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:689920ecfd60f992cafca4f5477d55720466ad2c7fa29bb56ac8d44a1ac2b47a", size = 249864, upload-time = "2025-08-29T15:33:17.434Z" }, - { url = "https://files.pythonhosted.org/packages/51/f8/d9d64e8da7bcddb094d511154824038833c81e3a039020a9d6539bf303e9/coverage-7.10.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ec98435796d2624d6905820a42f82149ee9fc4f2d45c2c5bc5a44481cc50db62", size = 251969, upload-time = "2025-08-29T15:33:18.822Z" }, - { url = "https://files.pythonhosted.org/packages/43/28/c43ba0ef19f446d6463c751315140d8f2a521e04c3e79e5c5fe211bfa430/coverage-7.10.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b37201ce4a458c7a758ecc4efa92fa8ed783c66e0fa3c42ae19fc454a0792153", size = 249659, upload-time = "2025-08-29T15:33:20.407Z" }, - { url = "https://files.pythonhosted.org/packages/79/3e/53635bd0b72beaacf265784508a0b386defc9ab7fad99ff95f79ce9db555/coverage-7.10.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2904271c80898663c810a6b067920a61dd8d38341244a3605bd31ab55250dad5", size = 247714, upload-time = "2025-08-29T15:33:21.751Z" }, - { url = "https://files.pythonhosted.org/packages/4c/55/0964aa87126624e8c159e32b0bc4e84edef78c89a1a4b924d28dd8265625/coverage-7.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5aea98383463d6e1fa4e95416d8de66f2d0cb588774ee20ae1b28df826bcb619", size = 248351, upload-time = "2025-08-29T15:33:23.105Z" }, - { url = "https://files.pythonhosted.org/packages/eb/ab/6cfa9dc518c6c8e14a691c54e53a9433ba67336c760607e299bfcf520cb1/coverage-7.10.6-cp311-cp311-win32.whl", hash = "sha256:e3fb1fa01d3598002777dd259c0c2e6d9d5e10e7222976fc8e03992f972a2cba", size = 219562, upload-time = "2025-08-29T15:33:24.717Z" }, - { url = "https://files.pythonhosted.org/packages/5b/18/99b25346690cbc55922e7cfef06d755d4abee803ef335baff0014268eff4/coverage-7.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:f35ed9d945bece26553d5b4c8630453169672bea0050a564456eb88bdffd927e", size = 220453, upload-time = "2025-08-29T15:33:26.482Z" }, - { url = "https://files.pythonhosted.org/packages/d8/ed/81d86648a07ccb124a5cf1f1a7788712b8d7216b593562683cd5c9b0d2c1/coverage-7.10.6-cp311-cp311-win_arm64.whl", hash = "sha256:99e1a305c7765631d74b98bf7dbf54eeea931f975e80f115437d23848ee8c27c", size = 219127, upload-time = "2025-08-29T15:33:27.777Z" }, { url = "https://files.pythonhosted.org/packages/26/06/263f3305c97ad78aab066d116b52250dd316e74fcc20c197b61e07eb391a/coverage-7.10.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5b2dd6059938063a2c9fee1af729d4f2af28fd1a545e9b7652861f0d752ebcea", size = 217324, upload-time = "2025-08-29T15:33:29.06Z" }, { url = "https://files.pythonhosted.org/packages/e9/60/1e1ded9a4fe80d843d7d53b3e395c1db3ff32d6c301e501f393b2e6c1c1f/coverage-7.10.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:388d80e56191bf846c485c14ae2bc8898aa3124d9d35903fef7d907780477634", size = 217560, upload-time = "2025-08-29T15:33:30.748Z" }, { url = "https://files.pythonhosted.org/packages/b8/25/52136173c14e26dfed8b106ed725811bb53c30b896d04d28d74cb64318b3/coverage-7.10.6-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:90cb5b1a4670662719591aa92d0095bb41714970c0b065b02a2610172dbf0af6", size = 249053, upload-time = "2025-08-29T15:33:32.041Z" }, @@ -254,11 +187,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/0c/50db5379b615854b5cf89146f8f5bd1d5a9693d7f3a987e269693521c404/coverage-7.10.6-py3-none-any.whl", hash = "sha256:92c4ecf6bf11b2e85fd4d8204814dc26e6a19f0c9d938c207c5cb0eadfcabbe3", size = 208986, upload-time = "2025-08-29T15:35:14.506Z" }, ] -[package.optional-dependencies] -toml = [ - { name = "tomli", marker = "python_full_version <= '3.11'" }, -] - [[package]] name = "cryptography" version = "45.0.7" @@ -292,18 +220,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/11/09700ddad7443ccb11d674efdbe9a832b4455dc1f16566d9bd3834922ce5/cryptography-45.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1993a1bb7e4eccfb922b6cd414f072e08ff5816702a0bdb8941c247a6b1b287c", size = 4561639, upload-time = "2025-09-01T11:14:35.343Z" }, { url = "https://files.pythonhosted.org/packages/71/ed/8f4c1337e9d3b94d8e50ae0b08ad0304a5709d483bfcadfcc77a23dbcb52/cryptography-45.0.7-cp37-abi3-win32.whl", hash = "sha256:18fcf70f243fe07252dcb1b268a687f2358025ce32f9f88028ca5c364b123ef5", size = 2926552, upload-time = "2025-09-01T11:14:36.929Z" }, { url = "https://files.pythonhosted.org/packages/bc/ff/026513ecad58dacd45d1d24ebe52b852165a26e287177de1d545325c0c25/cryptography-45.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:7285a89df4900ed3bfaad5679b1e668cb4b38a8de1ccbfc84b05f34512da0a90", size = 3392742, upload-time = "2025-09-01T11:14:38.368Z" }, - { url = "https://files.pythonhosted.org/packages/13/3e/e42f1528ca1ea82256b835191eab1be014e0f9f934b60d98b0be8a38ed70/cryptography-45.0.7-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:de58755d723e86175756f463f2f0bddd45cc36fbd62601228a3f8761c9f58252", size = 3572442, upload-time = "2025-09-01T11:14:39.836Z" }, - { url = "https://files.pythonhosted.org/packages/59/aa/e947693ab08674a2663ed2534cd8d345cf17bf6a1facf99273e8ec8986dc/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a20e442e917889d1a6b3c570c9e3fa2fdc398c20868abcea268ea33c024c4083", size = 4142233, upload-time = "2025-09-01T11:14:41.305Z" }, - { url = "https://files.pythonhosted.org/packages/24/06/09b6f6a2fc43474a32b8fe259038eef1500ee3d3c141599b57ac6c57612c/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:258e0dff86d1d891169b5af222d362468a9570e2532923088658aa866eb11130", size = 4376202, upload-time = "2025-09-01T11:14:43.047Z" }, - { url = "https://files.pythonhosted.org/packages/00/f2/c166af87e95ce6ae6d38471a7e039d3a0549c2d55d74e059680162052824/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d97cf502abe2ab9eff8bd5e4aca274da8d06dd3ef08b759a8d6143f4ad65d4b4", size = 4141900, upload-time = "2025-09-01T11:14:45.089Z" }, - { url = "https://files.pythonhosted.org/packages/16/b9/e96e0b6cb86eae27ea51fa8a3151535a18e66fe7c451fa90f7f89c85f541/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:c987dad82e8c65ebc985f5dae5e74a3beda9d0a2a4daf8a1115f3772b59e5141", size = 4375562, upload-time = "2025-09-01T11:14:47.166Z" }, - { url = "https://files.pythonhosted.org/packages/36/d0/36e8ee39274e9d77baf7d0dafda680cba6e52f3936b846f0d56d64fec915/cryptography-45.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c13b1e3afd29a5b3b2656257f14669ca8fa8d7956d509926f0b130b600b50ab7", size = 3322781, upload-time = "2025-09-01T11:14:48.747Z" }, - { url = "https://files.pythonhosted.org/packages/99/4e/49199a4c82946938a3e05d2e8ad9482484ba48bbc1e809e3d506c686d051/cryptography-45.0.7-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a862753b36620af6fc54209264f92c716367f2f0ff4624952276a6bbd18cbde", size = 3584634, upload-time = "2025-09-01T11:14:50.593Z" }, - { url = "https://files.pythonhosted.org/packages/16/ce/5f6ff59ea9c7779dba51b84871c19962529bdcc12e1a6ea172664916c550/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:06ce84dc14df0bf6ea84666f958e6080cdb6fe1231be2a51f3fc1267d9f3fb34", size = 4149533, upload-time = "2025-09-01T11:14:52.091Z" }, - { url = "https://files.pythonhosted.org/packages/ce/13/b3cfbd257ac96da4b88b46372e662009b7a16833bfc5da33bb97dd5631ae/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d0c5c6bac22b177bf8da7435d9d27a6834ee130309749d162b26c3105c0795a9", size = 4385557, upload-time = "2025-09-01T11:14:53.551Z" }, - { url = "https://files.pythonhosted.org/packages/1c/c5/8c59d6b7c7b439ba4fc8d0cab868027fd095f215031bc123c3a070962912/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:2f641b64acc00811da98df63df7d59fd4706c0df449da71cb7ac39a0732b40ae", size = 4149023, upload-time = "2025-09-01T11:14:55.022Z" }, - { url = "https://files.pythonhosted.org/packages/55/32/05385c86d6ca9ab0b4d5bb442d2e3d85e727939a11f3e163fc776ce5eb40/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:f5414a788ecc6ee6bc58560e85ca624258a55ca434884445440a810796ea0e0b", size = 4385722, upload-time = "2025-09-01T11:14:57.319Z" }, - { url = "https://files.pythonhosted.org/packages/23/87/7ce86f3fa14bc11a5a48c30d8103c26e09b6465f8d8e9d74cf7a0714f043/cryptography-45.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1f3d56f73595376f4244646dd5c5870c14c196949807be39e79e7bd9bac3da63", size = 3332908, upload-time = "2025-09-01T11:14:58.78Z" }, ] [[package]] @@ -328,7 +244,7 @@ test = [ [package.metadata] requires-dist = [ { name = "boto3" }, - { name = "gha-runner", git = "https://github.com/Open-Athena/gha-runner.git?rev=v1" }, + { name = "gha-runner", git = "https://github.com/omsf/gha-runner.git?rev=8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6" }, { name = "moto", extras = ["ec2"], marker = "extra == 'test'" }, { name = "pytest", marker = "extra == 'test'" }, { name = "pytest-cov", marker = "extra == 'test'" }, @@ -338,22 +254,10 @@ requires-dist = [ ] provides-extras = ["test"] -[[package]] -name = "exceptiongroup" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, -] - [[package]] name = "gha-runner" -version = "0.6.1.post19+ge8ef9e3" -source = { git = "https://github.com/Open-Athena/gha-runner.git?rev=v1#e8ef9e362e00d5be5a58a9af01091d784611bc05" } +version = "0.7.0" +source = { git = "https://github.com/omsf/gha-runner.git?rev=8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6#8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6" } dependencies = [ { name = "requests" }, ] @@ -403,26 +307,6 @@ version = "3.0.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357, upload-time = "2024-10-18T15:20:51.44Z" }, - { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393, upload-time = "2024-10-18T15:20:52.426Z" }, - { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732, upload-time = "2024-10-18T15:20:53.578Z" }, - { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866, upload-time = "2024-10-18T15:20:55.06Z" }, - { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964, upload-time = "2024-10-18T15:20:55.906Z" }, - { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977, upload-time = "2024-10-18T15:20:57.189Z" }, - { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366, upload-time = "2024-10-18T15:20:58.235Z" }, - { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091, upload-time = "2024-10-18T15:20:59.235Z" }, - { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065, upload-time = "2024-10-18T15:21:00.307Z" }, - { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514, upload-time = "2024-10-18T15:21:01.122Z" }, - { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353, upload-time = "2024-10-18T15:21:02.187Z" }, - { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392, upload-time = "2024-10-18T15:21:02.941Z" }, - { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984, upload-time = "2024-10-18T15:21:03.953Z" }, - { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120, upload-time = "2024-10-18T15:21:06.495Z" }, - { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032, upload-time = "2024-10-18T15:21:07.295Z" }, - { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057, upload-time = "2024-10-18T15:21:08.073Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359, upload-time = "2024-10-18T15:21:09.318Z" }, - { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306, upload-time = "2024-10-18T15:21:10.185Z" }, - { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094, upload-time = "2024-10-18T15:21:11.005Z" }, - { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521, upload-time = "2024-10-18T15:21:12.911Z" }, { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, @@ -517,12 +401,10 @@ version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } wheels = [ @@ -534,7 +416,7 @@ name = "pytest-cov" version = "6.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coverage", extra = ["toml"] }, + { name = "coverage" }, { name = "pluggy" }, { name = "pytest" }, ] @@ -561,24 +443,6 @@ version = "6.0.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, - { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, - { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, - { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, - { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, - { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, - { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, - { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, - { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, - { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, @@ -687,54 +551,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/9d/aef9ec5fd5a4ee2f6a96032c4eda5888c5c7cec65cef6b28c4fc37671d88/syrupy-4.9.1-py3-none-any.whl", hash = "sha256:b94cc12ed0e5e75b448255430af642516842a2374a46936dd2650cfb6dd20eda", size = 52214, upload-time = "2025-03-24T01:36:35.278Z" }, ] -[[package]] -name = "tomli" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, - { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, - { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, - { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, - { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, - { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, - { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, - { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, - { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, - { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, - { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, - { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, - { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, - { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, - { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, - { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, - { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, - { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, - { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, - { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, - { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, - { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, - { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, - { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, - { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, - { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, - { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, -] - [[package]] name = "urllib3" version = "2.5.0" From 23d1b354a3577c38ab972d186c01a5515f074a75 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Thu, 9 Jul 2026 22:21:37 -0400 Subject: [PATCH 69/69] Keep `gha_runner` on Open-Athena fork `@v1` (now upstream-merged) Reverts the source repoint to `omsf/gha-runner@8fbdacf`: switching straight to upstream would silently drop two OA-fork patches ec2-gha relies on (`Support multiple runners per instance`, `print PublicDnsName`). Instead, `omsf/gha-runner@8fbdacf` was merged into the OA fork and `v1` fast-forwarded to that merge, so `@v1` now carries upstream's improvements (GitHub API status in errors, py3.12 floor) plus the OA patches. `uv.lock` resolves `@v1` to Open-Athena/gha-runner@195de1b. Retains the `requires-python >= 3.12` bump and userdata `.rstrip()` from the prior commit. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- uv.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9740999..6a00a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Start an AWS GitHub Actions Runner" readme = "README.md" requires-python = ">=3.12" authors = [{ name = "Ethan Holz", email = "ethan.holz@omsf.io" }] -dependencies = ["boto3", "gha_runner @ git+https://github.com/omsf/gha-runner.git@8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6"] +dependencies = ["boto3", "gha_runner @ git+https://github.com/Open-Athena/gha-runner.git@v1"] [project.optional-dependencies] test = ["pytest", "pytest-cov", "moto[ec2]", "responses", "ruff", "syrupy"] diff --git a/uv.lock b/uv.lock index 314e993..c9cf1e9 100644 --- a/uv.lock +++ b/uv.lock @@ -244,7 +244,7 @@ test = [ [package.metadata] requires-dist = [ { name = "boto3" }, - { name = "gha-runner", git = "https://github.com/omsf/gha-runner.git?rev=8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6" }, + { name = "gha-runner", git = "https://github.com/Open-Athena/gha-runner.git?rev=v1" }, { name = "moto", extras = ["ec2"], marker = "extra == 'test'" }, { name = "pytest", marker = "extra == 'test'" }, { name = "pytest-cov", marker = "extra == 'test'" }, @@ -256,8 +256,8 @@ provides-extras = ["test"] [[package]] name = "gha-runner" -version = "0.7.0" -source = { git = "https://github.com/omsf/gha-runner.git?rev=8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6#8fbdacf05750dd12c652eb8cdf8d8adf76bb43e6" } +version = "0.6.1.post49+g195de1b" +source = { git = "https://github.com/Open-Athena/gha-runner.git?rev=v1#195de1b3cdcd42a6e90c5374d4509d112a96bdb2" } dependencies = [ { name = "requests" }, ]