diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03a3219..0991a8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,7 +105,7 @@ jobs: - name: Install tini run: apt-get update && apt-get install -y tini - - name: Build infrahub-backup binary + - name: Build binaries (infrahub-backup, infrahub-taskmanager, infrahub-collect) run: make build - name: Install Python dependencies @@ -161,7 +161,7 @@ jobs: - name: Install tini run: apt-get update && apt-get install -y tini - - name: Build infrahub-backup binary + - name: Build binaries (infrahub-backup, infrahub-taskmanager, infrahub-collect) run: make build - name: Install Python dependencies diff --git a/AGENTS.md b/AGENTS.md index 6850086..5df0cfc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,20 +4,21 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -`infrahub-ops-cli` is a Go-based toolset for managing and maintaining Infrahub instances. The project provides two specialized CLI binaries: +`infrahub-ops-cli` is a Go-based toolset for managing and maintaining Infrahub instances. The project provides three specialized CLI binaries: - **infrahub-backup** - Backup/restore operations and environment detection - **infrahub-taskmanager** - Task manager (Prefect) maintenance operations +- **infrahub-collect** - Troubleshooting-bundle collection (read-only diagnostics) for support -Both tools share common internal application logic but expose different commands through their respective main entry points. +All three tools share common internal application logic but expose different commands through their respective main entry points. ## Common Development Commands ### Building and Running -- `make build` - Build both binaries to `bin/infrahub-backup` and `bin/infrahub-taskmanager` -- `make build-all` - Cross-compile both binaries for Linux, Darwin, and Windows (amd64/arm64) -- `make install` - Build and install both binaries to `$GOPATH/bin` +- `make build` - Build all three binaries to `bin/infrahub-backup`, `bin/infrahub-taskmanager`, and `bin/infrahub-collect` +- `make build-all` - Cross-compile all three binaries for Linux, Darwin, and Windows (amd64/arm64) +- `make install` - Build and install all three binaries to `$GOPATH/bin` - `make clean` - Remove build artifacts ### Testing and Quality @@ -40,7 +41,7 @@ Whenever Go modules change (any modification to `go.mod` or `go.sum` — adding, ## Architecture -The codebase follows a command-pattern architecture using Cobra for CLI structure, with two separate binary entry points sharing common internal logic: +The codebase follows a command-pattern architecture using Cobra for CLI structure, with three separate binary entry points sharing common internal logic: ### Core Components @@ -54,50 +55,64 @@ The codebase follows a command-pattern architecture using Cobra for CLI structur - Commands: `flush flow-runs`, `flush stale-runs`, `environment detect`, `environment list`, `version` - Uses shared application logic from `src/internal/app` -3. **src/internal/app/app.go** - Core application logic +3. **src/cmd/infrahub-collect/main.go** - Troubleshooting-bundle tool entry point + - Defines root command with troubleshooting-bundle collection + - Commands: `create`, `environment detect`, `environment list`, `version` + - Uses shared application logic from `src/internal/app` + +4. **src/internal/app/app.go** - Core application logic - `InfrahubOps` struct - Main application controller - `CommandExecutor` - Handles Docker Compose and system command execution - Environment detection (Docker vs Kubernetes) - Docker project discovery and validation - - Shared by both CLI tools + - Shared by all three CLI tools -4. **src/internal/app/backup.go** - Backup and restore operations +5. **src/internal/app/backup.go** - Backup and restore operations - Creates tar.gz backups with metadata JSON - Backs up Neo4j database, PostgreSQL (task-manager), and artifacts - Implements safe backup with container stopping/starting - Restore validates metadata and handles version compatibility -5. **src/internal/app/taskmanager.go** - Task management operations +6. **src/internal/app/taskmanager.go** - Task management operations - PostgreSQL database connection management - Flow run cleanup operations (completed/failed/cancelled) - Stale run cancellation (stuck in running state) - Uses embedded Python scripts for Prefect API operations -6. **src/internal/app/utils.go** - Utility functions +7. **src/internal/app/collect*.go** - Troubleshooting-bundle collection + - `collect.go` - Orchestrator (`CollectBundle`), collector run plan, staging/archive lifecycle + - `collect_manifest.go` - `bundle_information.json` manifest and per-collector results + - `collect_logs.go` - Per-replica service log collector (backend-agnostic) + - `collect_diagnostics.go` - Database, message-queue, cache, task-worker, task-manager, and server collectors + - `collect_metrics.go` - Container resource metrics collector + - `collect_extras.go` - Opt-in `--include-backup` and `--benchmark` collectors + - `masking.go` - Key-name secret masking for env and config dumps + +8. **src/internal/app/utils.go** - Utility functions - File operations, checksum validation - Environment variable handling - Version detection and comparison -7. **src/internal/app/cli.go** - Shared CLI configuration +9. **src/internal/app/cli.go** - Shared CLI configuration - `ConfigureRootCommand()` - Sets up common flags and configuration - `AttachEnvironmentCommands()` - Adds environment detection commands - - Shared between both binaries + - Shared between all three binaries ### Key Design Patterns -- **Split Binary Architecture**: Two specialized binaries sharing common internal logic for focused functionality +- **Split Binary Architecture**: Three specialized binaries sharing common internal logic for focused functionality - **Embedded Scripts**: Python scripts are embedded using Go's embed package (src/internal/app/scripts directory) - **Docker Compose Integration**: All operations work through Docker Compose commands - **Project-based Operations**: Can target specific Docker Compose projects with `--project` flag - **Streaming Output**: Commands stream output in real-time for user feedback -- **Shared Configuration**: Both binaries use the same configuration system and environment variables +- **Shared Configuration**: All binaries use the same configuration system and environment variables ## Docker Compose Dependencies -Both tools assume Infrahub is deployed using Docker Compose with these service names: +All tools assume Infrahub is deployed using Docker Compose with these service names: -- `database` (Neo4j) - Used by infrahub-backup -- `task-manager-db` (PostgreSQL) - Used by both tools +- `database` (Neo4j) - Used by infrahub-backup and infrahub-collect +- `task-manager-db` (PostgreSQL) - Used by infrahub-backup and infrahub-taskmanager - `infrahub-server`, `task-worker`, `task-manager`, `task-manager-background-svc` - Application containers - `cache`, `message-queue` - Infrastructure services @@ -266,7 +281,10 @@ The codebase uses explicit error wrapping with `fmt.Errorf` for context. All com - Go 1.25.0 + kloset v1.0.13 (Plakar core), integration-fs (storage), cobra, logrus, viper (002-plakar-integration) - Plakar repository (local filesystem or S3 via integration backends) (002-plakar-integration) +- Go 1.25.0 + cobra, viper, logrus; Docker/Kubernetes via `docker`/`kubectl` CLI shell-out through `CommandExecutor` — no client-go or Docker SDK (003-collect-tool) +- Local filesystem bundle output under `--output-dir` (default `./infrahub_bundles`); no network egress beyond the target deployment (003-collect-tool) ## Recent Changes +- 003-collect-tool: Added `infrahub-collect` troubleshooting-bundle binary (collector framework, log/metrics primitives on both backends, key-name secret masking, bundle manifest) - 002-plakar-integration: Added kloset (Plakar core library), integration-fs (filesystem storage/exporter), cobra, logrus diff --git a/Makefile b/Makefile index 08ff85e..b22e1fd 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build build-all clean install test lint fmt vet help docker-build docker-build-multi docker-push # Variables -BINARIES=infrahub-backup infrahub-taskmanager +BINARIES=infrahub-backup infrahub-taskmanager infrahub-collect BUILD_DIR=$(shell pwd)/bin SRC_ROOT=./src VERSION?= diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..e897221 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,28 @@ +# Developer Documentation + +Durable engineering knowledge for `infrahub-ops-cli`, extracted from completed +feature specs (`specs/`) as they land. The lifecycle is +`specs/ → dev/{adr,knowledge,guidelines}/`, after which the spec is archived +under `specs/archive/`. + +- **adr/** — Architecture Decision Records: why a structural choice was made, and the alternatives rejected. +- **knowledge/** — How the system works today (descriptive). +- **guidelines/** — Prescriptive conventions to follow in future code. + +## Current ADRs + +- [0001-shell-out-to-docker-kubectl-clis.md](adr/0001-shell-out-to-docker-kubectl-clis.md) - Use the `docker`/`kubectl` CLIs instead of client libraries +- [0002-non-fatal-timeout-bounded-collectors.md](adr/0002-non-fatal-timeout-bounded-collectors.md) - Non-fatal, timeout-bounded collector framework +- [0003-read-only-collection.md](adr/0003-read-only-collection.md) - Collection never mutates the workload lifecycle +- [0004-key-name-secret-masking.md](adr/0004-key-name-secret-masking.md) - Key-name secret masking through a single choke-point +- [0005-uniform-bundle-and-manifest.md](adr/0005-uniform-bundle-and-manifest.md) - Single `.tar.gz` bundle + manifest, uniform across environments +- [0006-include-backup-reuses-createbackup.md](adr/0006-include-backup-reuses-createbackup.md) - `--include-backup` reuses `CreateBackup` non-interactively +- [0007-offline-by-default-opt-in-benchmark.md](adr/0007-offline-by-default-opt-in-benchmark.md) - Offline by default; benchmark/image pulls opt-in + +## Current Knowledge + +- [infrahub-collect.md](knowledge/infrahub-collect.md) - The troubleshooting-bundle tool: CLI, bundle layout, manifest, backend seam + +## Current Guidelines + +- [collectors.md](guidelines/collectors.md) - Conventions for writing and maintaining collect collectors diff --git a/dev/adr/0001-shell-out-to-docker-kubectl-clis.md b/dev/adr/0001-shell-out-to-docker-kubectl-clis.md new file mode 100644 index 0000000..8ba3c04 --- /dev/null +++ b/dev/adr/0001-shell-out-to-docker-kubectl-clis.md @@ -0,0 +1,25 @@ +# 1. Shell out to `docker`/`kubectl` CLIs instead of client libraries + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/research.md (R1) + +## Context + +`infrahub-collect` (and the existing `infrahub-backup`/`infrahub-taskmanager`) must operate against both Docker Compose and Kubernetes deployments of Infrahub. The tools are copied onto customer hosts and CI runners where no Go toolchain or package manager is available (constitution Principle III), and they must cross-compile `CGO_ENABLED=0` for linux/darwin/windows on amd64/arm64. + +## Decision + +All Docker and Kubernetes interaction shells out to the `docker` and `kubectl` binaries through the shared `CommandExecutor`. No Kubernetes `client-go`, no Docker Engine SDK. New collectors reuse the existing backend abstraction (`EnvironmentBackend`) and executor rather than introducing a client dependency. + +## Consequences + +- No new Go module dependencies for collect; the vendor hash and cross-compilation stay trivial. +- `kubectl` handles kubeconfig, contexts, and auth plugins (OIDC, cloud IAM) for free — none of that has to be reimplemented. +- Remote Docker contexts keep working because the `docker` CLI resolves them. +- The cost is that behaviour depends on the CLIs being present and on parsing their text/JSON output; absence is detected and reported with an actionable error (`ErrCLIUnavailable`). + +## Alternatives Considered + +- **`k8s.io/client-go`** — rejected: large dependency tree, auth-plugin complexity, and inconsistent with the existing backends. +- **Docker Engine API over the socket** — rejected: breaks remote-context workflows the `docker` CLI handles transparently. diff --git a/dev/adr/0002-non-fatal-timeout-bounded-collectors.md b/dev/adr/0002-non-fatal-timeout-bounded-collectors.md new file mode 100644 index 0000000..18bf78a --- /dev/null +++ b/dev/adr/0002-non-fatal-timeout-bounded-collectors.md @@ -0,0 +1,31 @@ +# 2. Non-fatal, timeout-bounded collector framework + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/research.md (R2) + +## Context + +The primary use case for a troubleshooting bundle is a **degraded** Infrahub instance — a service is down, a pod is crash-looping, an exec hangs. A collection tool that aborts on the first failure, or that hangs on a wedged container, is useless exactly when it is needed most (spec FR-009, SC-005). + +## Decision + +Collection is a fixed, ordered list of named `collector` units run sequentially by an orchestrator (`CollectBundle` / `runCollectPlan`). Each collector: + +- Runs every subprocess under a context timeout (`exec.CommandContext`): 60s for exec/status dumps, 5 minutes for log downloads and file copies. A timeout is recorded verbatim as `timed out after `. +- Is **non-fatal**: an error is caught, logged as a `WARN`, and recorded in the bundle manifest as `failed` (with a reason) or `skipped` (when the collector does not apply); the run continues and the command still exits 0 with a partial bundle. +- A panic in a collector is recovered and converted into a recorded `failed` outcome, so no single collector can abort the whole run. + +Only a missing environment, an unwritable output directory, or a failure to write the archive is fatal (exit 1). + +## Consequences + +- A bundle from a broken instance is still produced, and the manifest accounts for 100% of attempted collectors with an explicit outcome. +- The tool never hangs on a degraded dependency — timeouts convert a hang into a recorded failure. +- Sequential execution keeps the streamed progress readable and avoids hammering a fragile instance; wall time stays within the 5-minute budget (SC-001). + +## Alternatives Considered + +- **Parallel collectors** — rejected for v1: interleaved output, higher load on a degraded instance, and no evidence the time budget needs it. +- **Abort on first failure** — rejected: contradicts the degraded-instance use case. +- **Unbounded subprocesses** — rejected: a single wedged `exec`/`cp` would stall the entire run (the exact failure mode a troubleshooting tool must survive). diff --git a/dev/adr/0003-read-only-collection.md b/dev/adr/0003-read-only-collection.md new file mode 100644 index 0000000..33b1dde --- /dev/null +++ b/dev/adr/0003-read-only-collection.md @@ -0,0 +1,28 @@ +# 3. Collection is strictly read-only with respect to the workload lifecycle + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/spec.md (FR-010, SC-003) + +## Context + +`infrahub-backup` may stop and restart application containers while it snapshots the database (Neo4j Community). `infrahub-collect` is meant to be run against production instances at any time — including while they are degraded — and often by customers themselves. A diagnostic tool that could stop, restart, or scale a workload would be unsafe to hand out. + +## Decision + +`infrahub-collect` never stops, restarts, or scales any container, pod, or workload belonging to the deployment. It uses only read/exec operations (`pods/log`, `pods/exec`, `docker logs`, `docker exec`, `docker stats`, `kubectl top`). The stop/start/scale code paths in the shared backend are simply not reached by the collect plan. + +Two deliberate exceptions, both operating on the **tool's own** transient resources, not the deployment's: + +- `--benchmark` creates and then removes a short-lived benchmark container/pod it owns. +- `--include-backup` delegates to `CreateBackup`, whose documented behaviour may stop/restart app containers; this is opt-in and called out in the docs (see ADR 0006). + +## Consequences + +- Safe to run against a live or degraded production instance without risk of an outage attributable to the tool. +- Kubernetes RBAC for the tool needs only read/exec verbs — no write or scale permissions. +- Interrupt handling is simpler: there is no workload state to restore on SIGINT, only a staging directory to clean up. + +## Alternatives Considered + +- **Reuse the backup tool's stop/collect/start pattern for consistency** — rejected: it would make the tool unsafe for its core (production, degraded) use case and require elevated permissions. diff --git a/dev/adr/0004-key-name-secret-masking.md b/dev/adr/0004-key-name-secret-masking.md new file mode 100644 index 0000000..8a3c838 --- /dev/null +++ b/dev/adr/0004-key-name-secret-masking.md @@ -0,0 +1,32 @@ +# 4. Key-name secret masking through a single choke-point + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/research.md (R5) + +## Context + +A troubleshooting bundle captures environment variables and configuration dumps (server env, server API config, Redis `CONFIG GET`, RabbitMQ `environment`/`status`). These contain secrets. Support engineers receive these bundles, so plaintext secrets must not be written (spec FR-008). The prior Python tool masked by key name, and parity was required. + +## Decision + +A pure function masks values whose **key name** contains any of `pass`, `secret`, `token`, `key` (case-insensitive substring), replacing the value with `********`. Every dump that can carry secrets is routed through this single choke-point (`masking.go`), and the set of masked outputs is enumerated explicitly: + +1. `infrahub-server` environment (`maskEnvOutput`) +2. server API configuration dump (`maskJSON`, recurses through nested JSON) +3. Redis `CONFIG GET '*'` (`maskConfigPairs`) +4. RabbitMQ `rabbitmqctl environment` and `status` (`maskErlangConfig`) + +The token list is a documented **minimum**; `pass` was added over the spec's `password` to also catch `requirepass`/`default_pass`. Raw service logs are written as-is (parity with the prior tool; the docs warn users to review before sharing). + +## Consequences + +- A single, table-tested function governs masking; new sensitive dumps must be added to the enumerated list and routed through it, so masking cannot be silently skipped. +- Over-masking (a non-secret key that happens to contain `key`) is accepted as safe; under-masking is not. +- Known limitation: credentials embedded in a value under a non-matching key (e.g. a connection string `..._URL=postgres://user:pw@host`) are not masked. This is a documented parity limitation, not a regression. + +## Alternatives Considered + +- **Value-pattern scrubbing inside logs** — out of scope for v1 (documented); logs are copied verbatim. +- **Per-collector ad-hoc masking** — rejected: a single choke-point with an enumerated call-site list is enforceable; scattered masking rots. +- **Allowlist of safe keys** — rejected: inverts the prior tool's behaviour and breaks parity. diff --git a/dev/adr/0005-uniform-bundle-and-manifest.md b/dev/adr/0005-uniform-bundle-and-manifest.md new file mode 100644 index 0000000..399017f --- /dev/null +++ b/dev/adr/0005-uniform-bundle-and-manifest.md @@ -0,0 +1,31 @@ +# 5. Single `.tar.gz` bundle with a manifest, uniform across environments + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/research.md (R6, R7) + +## Context + +Support tooling and habits must transfer between Docker Compose and Kubernetes deployments without per-environment instructions (spec FR-006, SC-004). Support also needs a machine-readable record of what a bundle contains and which collectors failed, so a partial bundle from a degraded instance is still trustworthy (FR-007). + +## Decision + +A collection run produces one `support_bundle_.tar.gz` whose internal layout is **identical** across Docker and Kubernetes, with all members under a top-level `bundle/` directory. A `bundle_information.json` manifest at the bundle root is the collect-side sibling of `BackupMetadata`: + +- Fields: `manifest_version` (date-serial constant, bumped on breaking change), `collect_id`, `created_at` (RFC3339 UTC), `tool_version`, `infrahub_version`, `environment` (`docker`|`kubernetes`), `log_lines`, and `collectors[]` with `{name, status, reason?, artifact?}`. +- `status` is a typed enum (`success`|`failed`|`skipped`); `reason` is required whenever status is not `success`; `artifact` appears only on success (e.g. the `--include-backup` path). +- Written last, after every collector has reported, so it reflects final outcomes. + +Files are staged in a temp directory created **inside** the output directory, packaged with the existing `createTarball`, and the staging dir is removed on success, failure, and interrupt (SIGINT/SIGTERM) so no temp files leak. + +## Consequences + +- One layout and one manifest schema to learn; the manifest is authoritative for what was attempted vs. produced. +- The manifest mirrors the proven `BackupMetadata` conventions (date-serial version constant, `MarshalIndent`, `os.WriteFile`). +- Staging inside the output directory keeps all writes on one filesystem and confined to the user-chosen location; a write failure (e.g. disk full) is a hard error after cleanup. + +## Alternatives Considered + +- **Streaming tar writer without a staging directory** — rejected: the manifest must be finalized *after* all collectors report, which a streaming writer cannot retro-fit. +- **Staging in `/tmp`** — rejected: cross-device writes and files leaking outside the user-designated directory. +- **Extending `BackupMetadata` rather than a sibling type** — rejected: different lifecycle and fields; the spec defines the manifest as a sibling. diff --git a/dev/adr/0006-include-backup-reuses-createbackup.md b/dev/adr/0006-include-backup-reuses-createbackup.md new file mode 100644 index 0000000..f2df368 --- /dev/null +++ b/dev/adr/0006-include-backup-reuses-createbackup.md @@ -0,0 +1,28 @@ +# 6. `--include-backup` reuses `CreateBackup` as a standalone artifact, invoked non-interactively + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/research.md (R10) + +## Context + +Support frequently asks for "logs plus a backup" so they can reproduce a customer issue locally (spec US3, FR-014). The backup logic already exists in `CreateBackup` and carries the operational-safety guarantees of constitution Principle II (metadata, checksums, restart-what-you-stopped). Collection must reuse it, not reimplement it. + +## Decision + +`--include-backup` runs a collector that delegates to the existing `CreateBackup` unmodified. The produced backup is a **standalone** `infrahub_backup_*.tar.gz` in the standard backup directory — referenced by the manifest's `artifact` field, **not embedded** in the bundle. The collector runs last in the plan (after every read-only collector), because the delegated backup may stop/restart app containers and must not taint the diagnostics. A backup failure is non-fatal: the bundle is still produced (US3 scenario 2). + +The backup is invoked with `force=true`. Collection is non-interactive and designed never to hang; `CreateBackup(force=false)` runs `waitForRunningTasks`, an unbounded loop that returns only when no Prefect tasks are running/pending, which never fully drains on a busy instance (e.g. the Enterprise edition's recurring background tasks). `--force` skips only that consistency gate; it does not weaken the backup's own integrity guarantees. + +## Consequences + +- Backups keep their full metadata/checksum guarantees; no divergent "lite" backup path to maintain. +- The bundle tarball stays small; the backup is discoverable by the standard backup tooling at its normal location. +- The backup runs to completion non-interactively rather than blocking on an unbounded task-drain wait — consistent with the tool's "never hang" contract. +- `CreateBackup` does not return the path it wrote, so the artifact is identified by diffing the backup directory listing around the call. + +## Alternatives Considered + +- **Embed the backup archive inside the bundle** — rejected: doubles disk usage and breaks the backup tool's own restore discovery (it expects the standard location/naming). +- **`force=false` to preserve the running-tasks wait** — rejected after it caused the Enterprise e2e to hang/fail with no artifact; the wait is a consistency nicety, not a safety guarantee, and `--force` is the supported non-interactive path. +- **A lighter read-only reproduction backup** — rejected: violates "reuse unmodified" and Principle II. diff --git a/dev/adr/0007-offline-by-default-opt-in-benchmark.md b/dev/adr/0007-offline-by-default-opt-in-benchmark.md new file mode 100644 index 0000000..c8c2566 --- /dev/null +++ b/dev/adr/0007-offline-by-default-opt-in-benchmark.md @@ -0,0 +1,25 @@ +# 7. Offline by default; image pulls and the benchmark are strictly opt-in + +**Status**: Accepted +**Date**: 2026-07-09 +**Source**: specs/archive/003-collect-tool/research.md (R11, R12), spec FR-012/FR-013 + +## Context + +Troubleshooting bundles are often collected in air-gapped or egress-restricted environments. The tool must work with only the operator's existing Docker/kubectl access, and any feature that needs the network must fail gracefully rather than block the whole collection. + +## Decision + +By default, collection performs **no** container image pulls and **no** network access beyond the target deployment itself. The only feature that reaches out is the opt-in `--benchmark`, which pulls and runs the OpsMill benchmark image (`registry.opsmill.io/opsmill/bench`, overridable via `INFRAHUB_BENCHMARK_IMAGE`). When the image cannot be pulled or run — e.g. air-gapped — the benchmark collector records `skipped` with a warning and the rest of the collection proceeds normally (it is a skip, not a failure). + +## Consequences + +- The default run is safe in air-gapped environments; nothing silently reaches the internet. +- Opting into the benchmark is an explicit, documented choice, and its failure degrades gracefully. +- The override env var keeps air-gapped-with-private-registry workflows possible. + +## Alternatives Considered + +- **Benchmark on by default (as the prior Python tool)** — rejected: offline-by-default is the correct posture for an ops tool that runs in restricted networks. +- **A Go-native benchmark to avoid the image pull** — rejected: duplicates the maintained benchmark suite. +- **Treat a benchmark pull failure as `failed`** — rejected: an unavailable image in an air-gapped run is expected, not an error; `skipped` keeps the manifest honest without alarming the operator. diff --git a/dev/guidelines/collectors.md b/dev/guidelines/collectors.md new file mode 100644 index 0000000..1bb8556 --- /dev/null +++ b/dev/guidelines/collectors.md @@ -0,0 +1,47 @@ + + +# Collector Guidelines + +## Overview + +Conventions for writing and maintaining `infrahub-collect` collectors (`src/internal/app/collect*.go`). A collector gathers one category of diagnostic data into the staging bundle. These rules keep collection safe on degraded instances and keep the bundle trustworthy. Background: [ADR 0002](../adr/0002-non-fatal-timeout-bounded-collectors.md), [ADR 0003](../adr/0003-read-only-collection.md), [ADR 0004](../adr/0004-key-name-secret-masking.md). + +## Time-bound every subprocess + +A collector runs against a possibly-wedged container. Never call the context-free `runCommand`/`Exec` on the collect path — a hang there stalls the whole run. Use the bounded variants and the standard bounds: + +- exec / status dumps → `collectExecTimeout` (60s) +- log downloads and file copies → `collectTransferTimeout` (5 min) + +A timeout must surface as a `*timeoutError` so the orchestrator records the manifest reason verbatim as `timed out after `. When aggregating several sub-commands into one collector error, `%w`-wrap a sub-command timeout so `errors.As` still finds it — do not flatten it into a `%v` string. + +## Failures are non-fatal; distinguish absent from failed + +Return a wrapped error on failure; the orchestrator logs a `WARN` and records the collector as `failed` with the reason. Never call `os.Exit` or print errors directly (constitution Principle V). The run must continue and still exit 0. + +- **Service not deployed** → record `skipped` with a reason (via the skip precondition). A genuine "nothing matched" enumeration returns `(nil, nil)`. +- **Service present but the command failed** → record `failed`. Do not collapse an execution error into "not deployed"; that produces a misleadingly-empty bundle. + +Do not mark a collector `success` when it collected nothing due to a swallowed error — an empty file that looks intentional is worse than a recorded failure. + +## Route sensitive output through the masking choke-point + +Any env or configuration dump that can carry secrets MUST pass through `masking.go` before it is written, and MUST be added to the enumerated masked-output set (see [ADR 0004](../adr/0004-key-name-secret-masking.md)). Pick the matching helper: `maskEnvOutput` (KEY=VALUE), `maskConfigPairs` (Redis alternating lines), `maskErlangConfig` (rabbitmqctl tuples), `maskJSON` (JSON). Never write a new sensitive dump without masking it. + +## Stay read-only + +A collector must never stop, restart, or scale a deployment workload. Use only read/exec operations. The sole permitted mutations are the tool's **own** transient resources (the `--benchmark` container/pod it creates and deletes) and the opt-in `--include-backup` delegation. + +## Don't mutate shared backend helpers — add bounded variants + +`EnvironmentBackend` helpers (`CopyFrom`, `Exec`, `getPodForService`, `getInfrahubVersion`, …) are shared with `infrahub-backup`/`infrahub-taskmanager`. When collect needs a timeout-bounded or otherwise different behaviour, **add a collect-only variant** (e.g. `CopyFromContext`, `getPodForServiceContext`, `ExecContext`) rather than changing the shared method's signature or behaviour. Keep compile-time `var _ collectBackend = (*DockerBackend)(nil)` / `(*KubernetesBackend)(nil)` assertions so a seam change fails at build time, not at runtime. + +## Non-interactive invocations must not gate on unbounded waits + +Anything collect calls must complete without waiting on an operator or on an open-ended condition. When reusing an interactive path, pass the non-interactive flag: `--include-backup` calls `CreateBackup(force=true, …)` so it skips the unbounded `waitForRunningTasks` loop (see [ADR 0006](../adr/0006-include-backup-reuses-createbackup.md)). + +## Testing + +- Unit-test pure logic table-driven (masking, manifest shape, filename derivation, arg construction, replica parsing) — no mocking framework; use a fake `collectBackend`. +- Cover the failure branches: a collector that fails still exits 0 and records `failed`; a read-only guard test asserts no `Stop`/`Start`/scale is called across the full plan. +- Container-touching flows are covered by the `-m docker` / `-m k8s` pytest e2e suites (Docker and Kubernetes) in CI. Note the CI edition matrix (`community`/`enterprise`): behaviours that differ by Neo4j edition (e.g. the backup path) must be validated on both. diff --git a/dev/knowledge/infrahub-collect.md b/dev/knowledge/infrahub-collect.md new file mode 100644 index 0000000..d4df8a0 --- /dev/null +++ b/dev/knowledge/infrahub-collect.md @@ -0,0 +1,71 @@ + + +# Infrahub Collect + +## Overview + +`infrahub-collect` is the third CLI binary in this repo. It gathers a **troubleshooting bundle** — service logs, diagnostic status, configuration, and metrics — from a Docker Compose or Kubernetes Infrahub deployment into a single local archive for OpsMill support. It replaces the Python `invoke bundle collect` script and adds first-class Kubernetes support. Collection is strictly read-only (see [ADR 0003](../adr/0003-read-only-collection.md)). + +Like the other binaries, the entry point (`src/cmd/infrahub-collect`) is thin Cobra wiring; all logic lives in `src/internal/app` (`collect*.go`, `masking.go`). + +## CLI surface + +```text +infrahub-collect [global-flags] + create Collect a troubleshooting bundle + environment detect Detect the deployment environment (shared) + environment list List detected Docker projects / K8s namespaces (shared) + version Print the build version (shared) +``` + +Flags and their `INFRAHUB_*` environment equivalents: + +| Flag | Default | Env | Scope | +|------|---------|-----|-------| +| `--project` | auto-detect | `INFRAHUB_PROJECT` | global (shared) | +| `--k8s-namespace` | auto-detect | `INFRAHUB_K8S_NAMESPACE` | global (shared) | +| `--output-dir` | `./infrahub_bundles` | `INFRAHUB_OUTPUT_DIR` | global (collect) | +| `--log-format` | `text` | `INFRAHUB_LOG_FORMAT` | global (shared) | +| `--log-lines` | `100000` | `INFRAHUB_LOG_LINES` | `create` | +| `--include-backup` | `false` | `INFRAHUB_INCLUDE_BACKUP` | `create` | +| `--include-queries` | `false` | `INFRAHUB_INCLUDE_QUERIES` | `create` | +| `--benchmark` | `false` | `INFRAHUB_BENCHMARK` | `create` | + +`create` exits 0 whenever the archive is produced — including a partial bundle with failed collectors. It exits 1 only for no usable environment, an unwritable output directory, or an archiving failure. + +## Bundle layout + +Output: `/support_bundle_.tar.gz`, everything under a top-level `bundle/`, identical on Docker and Kubernetes: + +```text +bundle/ +├── bundle_information.json # manifest (see below) +├── logs// # one file per replica; *.previous.log for restarted k8s containers +├── database/ # Neo4j neo4j.log, debug.log (+ full /logs with --include-queries) +├── message-queue/ # RabbitMQ queues/exchanges/bindings/connections/channels/status/environment +├── cache/ # Redis info/clients/config(masked)/slowlog/dbsize +├── task-worker// # Prefect worker state per replica +├── task-manager/ # work pools, queues, flow runs, events, automations +├── server/ # version, pip list, API info/config/schema, env (masked) +├── metrics/ # docker stats / kubectl top +└── benchmark/ # only when --benchmark ran successfully +``` + +## Manifest (`bundle_information.json`) + +The collect-side sibling of `BackupMetadata`. Contract: `specs/archive/003-collect-tool/contracts/manifest.schema.json`. Fields: `manifest_version` (date-serial), `collect_id`, `created_at` (RFC3339 UTC), `tool_version`, `infrahub_version`, `environment` (`docker`|`kubernetes`), `log_lines`, and `collectors[]` of `{name, status, reason?, artifact?}` where `status` ∈ `success|failed|skipped`. The manifest accounts for every planned collector and is written last so it reflects final outcomes. See [ADR 0005](../adr/0005-uniform-bundle-and-manifest.md). + +## The collect backend seam + +Collectors consume a narrow interface on top of the shared `EnvironmentBackend`, satisfied by both `DockerBackend` and `KubernetesBackend` (compile-time `var _ collectBackend` assertions) and by a test fake: + +- `ServiceReplicas(service) ([]Replica, error)` — one `Replica` per running unit. On Docker, one per container (by container **name**); on Kubernetes, one per **pod container** (multi-container pods such as CNPG or sidecar-bearing pods yield several), each carrying its own `restartCount`. A genuine "no pods matched" is an empty, non-error result (→ `skipped`); a kubectl execution failure is a real error (→ `failed`). +- `ReplicaLogs(replica, tailLines, previous)` — streams a replica's logs; `previous` fetches the prior container's logs (Kubernetes only, and only when that container restarted). +- `Metrics()` — one-shot `docker stats --no-stream` / `kubectl top pods`. + +Every subprocess on the collect path is timeout-bounded (see [ADR 0002](../adr/0002-non-fatal-timeout-bounded-collectors.md)); shared unbounded helpers used by backup keep collect-only bounded variants (`CopyFromContext`, `getPodForServiceContext`). + +## Gotchas + +- **Docker services log to stderr.** `docker logs` demuxes a container's output onto the matching process streams, and Infrahub services write to stderr. Capturing stdout alone yields near-empty logs, so the Docker log path uses a **combined** stdout+stderr pipe (`runCommandCombinedPipeContext`). The Kubernetes `kubectl logs` path needs no such merge. +- **A stopped-but-deployed Docker service** still has its logs collected and is recorded `failed` (not `skipped`); `skipped` is reserved for a service that is not deployed at all. diff --git a/docs/docs/guides/collect-troubleshooting-bundle.mdx b/docs/docs/guides/collect-troubleshooting-bundle.mdx index 90b5101..94ad797 100644 --- a/docs/docs/guides/collect-troubleshooting-bundle.mdx +++ b/docs/docs/guides/collect-troubleshooting-bundle.mdx @@ -222,7 +222,7 @@ For performance investigations, support may ask you to include a benchmark run: infrahub-collect create --benchmark ``` -The benchmark requires downloading the OpsMill benchmark container image and generates load against your instance. If the image cannot be pulled — for example in an air-gapped environment — the benchmark is skipped with a warning and the rest of the collection completes normally. +The benchmark requires downloading the OpsMill benchmark container image and measures whether the host meets Infrahub's resource requirements — disk IOPS, memory, and single-core CPU — rather than generating load against your instance. If the image cannot be pulled — for example in an air-gapped environment — the benchmark is skipped with a warning and the rest of the collection completes normally. ## Troubleshoot collection diff --git a/docs/docs/guides/install-collect.mdx b/docs/docs/guides/install-collect.mdx index 277ba31..7dd730d 100644 --- a/docs/docs/guides/install-collect.mdx +++ b/docs/docs/guides/install-collect.mdx @@ -24,7 +24,7 @@ Before installing Infrahub Collect, ensure you have: **If building from source:** - Git and network access to clone the repository -- Go 1.21 or later installed +- Go 1.25 or later installed ## Installation methods @@ -80,7 +80,7 @@ If you need a specific version or architecture, manually select the appropriate To install Infrahub Collect, build from source: -1. Install Go 1.21 or later: +1. Install Go 1.25 or later: ```bash # Check if Go is installed diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx index 6d413b9..f192e4b 100644 --- a/docs/docs/reference/configuration.mdx +++ b/docs/docs/reference/configuration.mdx @@ -44,6 +44,12 @@ Tools can be configured through these methods, applied in precedence order: |----------|-------------|---------|---------| | `PREFECT_API_DATABASE_CONNECTION_URL` | PostgreSQL connection string | Auto-detect | `postgresql://user:pass@localhost/prefect` | +### Collect configuration + +| Variable | Description | Default | Example | +|----------|-------------|---------|---------| +| `INFRAHUB_BENCHMARK_IMAGE` | Container image used by the opt-in `--benchmark` run | `registry.opsmill.io/opsmill/bench` | `registry.example.com/opsmill/bench` | + ## Command-line flag reference ### Global flags diff --git a/flake.nix b/flake.nix index ce6066e..53a5590 100644 --- a/flake.nix +++ b/flake.nix @@ -76,11 +76,21 @@ }; }); + infrahub-collect = pkgs.buildGoModule (commonAttrs // { + pname = "infrahub-collect"; + subPackages = [ "src/cmd/infrahub-collect" ]; + meta = commonAttrs.meta // { + description = "Troubleshooting bundle collection tool for Infrahub instances"; + mainProgram = "infrahub-collect"; + }; + }); + default = pkgs.symlinkJoin { name = "infrahub-ops-cli-${version}"; paths = [ self.packages.${system}.infrahub-backup self.packages.${system}.infrahub-taskmanager + self.packages.${system}.infrahub-collect ]; }; }; diff --git a/specs/archive/003-collect-tool/EXTRACTED.md b/specs/archive/003-collect-tool/EXTRACTED.md new file mode 100644 index 0000000..e1932c1 --- /dev/null +++ b/specs/archive/003-collect-tool/EXTRACTED.md @@ -0,0 +1,26 @@ +# Extraction Record + +**Extracted on**: 2026-07-09 +**Extracted by**: speckit.opsmill.extract + +## ADRs Created + +- dev/adr/0001-shell-out-to-docker-kubectl-clis.md (from R1) +- dev/adr/0002-non-fatal-timeout-bounded-collectors.md (from R2, critique E1) +- dev/adr/0003-read-only-collection.md (from spec FR-010 / SC-003) +- dev/adr/0004-key-name-secret-masking.md (from R5, critique E3) +- dev/adr/0005-uniform-bundle-and-manifest.md (from R6, R7) +- dev/adr/0006-include-backup-reuses-createbackup.md (from R10 + the force=true fix) +- dev/adr/0007-offline-by-default-opt-in-benchmark.md (from R11, R12, FR-012/FR-013) + +## Knowledge Updated + +- dev/knowledge/infrahub-collect.md (new: CLI surface, bundle layout, manifest, collect backend seam, gotchas) + +## Guidelines Updated + +- dev/guidelines/collectors.md (new: timeouts, non-fatal failures, masking choke-point, read-only, bounded shared-helper variants, testing) + +## Archive + +Spec directory moved to `specs/archive/003-collect-tool/` as a historical record. diff --git a/specs/003-collect-tool/checklists/requirements.md b/specs/archive/003-collect-tool/checklists/requirements.md similarity index 100% rename from specs/003-collect-tool/checklists/requirements.md rename to specs/archive/003-collect-tool/checklists/requirements.md diff --git a/specs/archive/003-collect-tool/contracts/bundle-layout.md b/specs/archive/003-collect-tool/contracts/bundle-layout.md new file mode 100644 index 0000000..92e1794 --- /dev/null +++ b/specs/archive/003-collect-tool/contracts/bundle-layout.md @@ -0,0 +1,43 @@ +# Bundle Layout Contract + +**Status**: normative — published in `docs/docs/guides/collect-troubleshooting-bundle.mdx` (Step 4). The layout is identical for Docker Compose and Kubernetes deployments (FR-006, SC-004). + +## Archive + +- Filename: `support_bundle_.tar.gz` +- Compression: gzip; created with the existing `createTarball` helper +- All members live under a single top-level `bundle/` directory + +## Directory tree + +```text +bundle/ +├── bundle_information.json # Manifest — see manifest.schema.json +├── logs/ +│ └── / # One directory per Infrahub service present +│ ├── .log # One file per replica — Docker: container name; +│ │ # k8s: .log, or _.log for +│ │ # multi-container pods +│ └── .previous.log # Kubernetes only, per container with restarts +├── database/ # Neo4j server logs: neo4j.log, debug.log +│ # (full log dir incl. query logs with --include-queries) +├── message-queue/ # RabbitMQ: queues, exchanges, bindings, connections, +│ # channels, status, environment (one file per dump) +├── cache/ # Redis: info, client list, configuration (masked), +│ # slow log, database size +├── task-worker/ +│ └── / # One directory per replica: Prefect worker CLI output +├── task-manager/ # Work pools, work queues, recent flow runs, events, +│ # automations +├── server/ # Infrahub version, installed packages, API info / +│ # config / schema, environment variables (masked) +├── metrics/ # docker stats / kubectl top output +└── benchmark/ # Present only when --benchmark ran successfully +``` + +## Rules + +- Directories for collectors that failed or were skipped MAY be absent or partially populated; the manifest is the authoritative record of what was attempted (FR-007). +- `logs//` uses per-replica filenames so multi-replica services never interleave (FR-002). +- Env/config dumps are masked before writing (FR-008); raw service logs are written as-is. +- A backup produced by `--include-backup` is **not** inside the archive; it is a standard backup artifact next to the bundle, referenced in the manifest (research R10). diff --git a/specs/archive/003-collect-tool/contracts/cli.md b/specs/archive/003-collect-tool/contracts/cli.md new file mode 100644 index 0000000..32c1c90 --- /dev/null +++ b/specs/archive/003-collect-tool/contracts/cli.md @@ -0,0 +1,57 @@ +# CLI Contract: infrahub-collect + +**Status**: normative — already published in `docs/docs/reference/commands.mdx`, `docs/docs/guides/collect-troubleshooting-bundle.mdx`, and `docs/docs/guides/install-collect.mdx`. The implementation must match this surface exactly. + +## Command structure + +```text +infrahub-collect [global-flags] [flags] + +Commands: + create Collect a troubleshooting bundle + environment detect Detect the deployment environment (shared, identical to infrahub-backup) + environment list List detected Docker projects / Kubernetes namespaces (shared) + version Print the build version (shared pattern: "Version: ") +``` + +## Global flags (persistent on root) + +| Flag | Default | Env var | Notes | +|------|---------|---------|-------| +| `--project ` | auto-detect | `INFRAHUB_PROJECT` | shared via `ConfigureRootCommand` | +| `--k8s-namespace ` | auto-detect | `INFRAHUB_K8S_NAMESPACE` | shared via `ConfigureRootCommand` | +| `--output-dir ` | `./infrahub_bundles` | `INFRAHUB_OUTPUT_DIR` | collect-specific persistent flag | +| `--log-format ` | `text` | `INFRAHUB_LOG_FORMAT` | shared via `ConfigureRootCommand` | +| `--help, -h` | — | — | cobra built-in | + +## `create` flags + +| Flag | Default | Env var | +|------|---------|---------| +| `--log-lines ` | `100000` | `INFRAHUB_LOG_LINES` | +| `--include-backup` | `false` | `INFRAHUB_INCLUDE_BACKUP` | +| `--include-queries` | `false` | `INFRAHUB_INCLUDE_QUERIES` | +| `--benchmark` | `false` | `INFRAHUB_BENCHMARK` | + +Precedence: flag > environment variable > default (viper, `INFRAHUB_` prefix, `-`→`_` replacement — existing shared behavior). + +## Behavioral contract for `create` + +1. Detects the environment (Docker Compose first unless `--k8s-namespace` is set), honoring `--project`/`--k8s-namespace`; multiple candidates without a selector → same selection behavior/error as `infrahub-backup` (FR-001). +2. Never stops, restarts, or scales any container/pod/workload (FR-010). The only exception is a transient benchmark container/pod the tool itself creates under `--benchmark`. +3. Runs all collectors even when some fail; each failure logs a `WARN` line and is recorded in the manifest (FR-009). +4. Every collector subprocess is time-bounded (60s for exec/status dumps, 5 min for log/copy transfers); a timeout is a per-collector failure (`timed out after ` in the manifest), never a hang. +5. Streams one `INFO` progress line per collector as it runs (Principle V); the docs' Step-3 transcript is the reference format. +6. Writes `support_bundle_.tar.gz` into the output directory and prints its path **and size** on success. Write failures (unwritable/full output directory) are hard errors (exit 1) after staging cleanup. +6. Performs no image pulls and no network egress beyond the deployment unless `--benchmark` is set (FR-012/FR-013). + +## Exit codes + +| Code | Condition | +|------|-----------| +| 0 | Archive produced — including partial bundles with failed collectors (FR-009) | +| 1 | Hard failure: no usable environment (neither docker nor kubectl usable → existing `ErrCLIUnavailable`/`ErrEnvironmentNotFound` messages), no deployment found, output dir unwritable, or archiving itself failed | + +## Interrupt behavior + +SIGINT/SIGTERM: no workload state to restore (read-only); staging directory is removed; no partial archive is left behind with a final name (edge case "Interrupted collection"). diff --git a/specs/archive/003-collect-tool/contracts/manifest.schema.json b/specs/archive/003-collect-tool/contracts/manifest.schema.json new file mode 100644 index 0000000..873153f --- /dev/null +++ b/specs/archive/003-collect-tool/contracts/manifest.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://opsmill.io/schemas/infrahub-collect/bundle_information.json", + "title": "Infrahub Collect bundle manifest (bundle_information.json)", + "type": "object", + "required": [ + "manifest_version", + "collect_id", + "created_at", + "tool_version", + "infrahub_version", + "environment", + "log_lines", + "collectors" + ], + "properties": { + "manifest_version": { + "type": "integer", + "description": "Date-serial manifest schema version, e.g. 2026070200" + }, + "collect_id": { + "type": "string", + "pattern": "^[0-9]{8}_[0-9]{6}$", + "description": "UTC timestamp identifier, shared with the archive filename" + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "RFC3339 UTC creation timestamp" + }, + "tool_version": { + "type": "string", + "description": "infrahub-collect build revision" + }, + "infrahub_version": { + "type": "string", + "description": "Detected Infrahub version; empty when the server was unreachable" + }, + "environment": { + "type": "string", + "enum": ["docker", "kubernetes"] + }, + "log_lines": { + "type": "integer", + "minimum": 1, + "description": "Effective per-container log line cap for this run" + }, + "collectors": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["name", "status"], + "properties": { + "name": { + "type": "string", + "description": "Stable collector identifier, e.g. logs/infrahub-server, cache-status, benchmark" + }, + "status": { + "type": "string", + "enum": ["success", "failed", "skipped"] + }, + "reason": { + "type": "string", + "description": "Required when status is failed or skipped" + }, + "artifact": { + "type": "string", + "description": "Optional path of a filesystem artifact the collector produced outside the bundle (e.g. the --include-backup archive); only present on success" + } + }, + "if": { + "properties": { "status": { "enum": ["failed", "skipped"] } } + }, + "then": { + "required": ["name", "status", "reason"] + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/specs/archive/003-collect-tool/critiques/critique-20260703-170700.md b/specs/archive/003-collect-tool/critiques/critique-20260703-170700.md new file mode 100644 index 0000000..48da521 --- /dev/null +++ b/specs/archive/003-collect-tool/critiques/critique-20260703-170700.md @@ -0,0 +1,92 @@ +# Critique Report: Infrahub Collect (003-collect-tool) + +**Date**: 2026-07-03 | **Artifacts reviewed**: spec.md, plan.md, research.md, data-model.md, contracts/ (cli.md, bundle-layout.md, manifest.schema.json), quickstart.md, constitution v1.1.0 + +## Executive Summary + +The spec and plan are in strong shape: the problem is well-evidenced (Jira INFP-415, grilled with the product owner), scope is disciplined (upload deferred to INFP-581, benchmark opt-in, log-scrubbing explicitly out), the constitution conflict was resolved ahead of time (v1.1.0 amendment), and the plan reuses the existing environment abstraction instead of inventing a parallel one. The published docs double as a precise, testable contract. + +One must-address engineering gap was found: **no timeout strategy for exec-based collectors**, which directly threatens the tool's core promise — because the primary use case is a *degraded* instance, a single hung `kubectl exec`/`docker exec` (wedged container, dead kubelet) would hang the entire run indefinitely, violating FR-009's spirit and SC-001's 5-minute budget. This is resolvable with a bounded plan/research update; no rethink needed. + +**Verdict: ⚠️ PROCEED WITH UPDATES** — apply the E1 fix (and the low-risk recommendations) to plan.md/research.md, then proceed to `/speckit-tasks`. + +## Product Lens Findings + +### 3a. Problem Validation + +Sound. The Kubernetes gap is concrete (support pulls logs pod-by-pod via kubectl/Rancher/GKE/Argo, inconsistently per customer), the Docker path has a real cost today (repo checkout + Python env), and the source is a groomed Jira ticket with a discovery brief. Scope boundaries (no upload, no Ansible targets, no log analysis) are explicit. No findings. + +### 3b. User Value Assessment + +- User stories are independently testable and correctly prioritized (K8s first — the zero-alternative case). MVP = US1 alone is viable. +- **P2 💡 (Replica file naming)**: The bundle layout contract names per-replica log files by "container ID / pod name". Pod names are meaningful; bare Docker container ID hexes are not — support engineers reading a bundle can't tell which replica is which without cross-referencing. *Suggestion*: on Docker, name files by container name (or `-`), keeping pod names on Kubernetes. + +### 3c. Alternative Approaches + +Considered and settled during grilling (third binary vs. subcommand; parity-minus-benchmark). Reuse of existing backends beats any off-the-shelf tool (must understand Infrahub's service topology, CNPG primaries, Prefect internals). Cost of inaction is ongoing support toil. No findings. + +### 3d. Edge Cases & User Experience + +- Degraded-instance behavior, missing CLIs, absent optional services, air-gapped mode, and interrupts are all specified — unusually thorough. +- **P1 💡 (Disk-space failure & bundle size)**: Nothing specifies behavior when the output directory runs out of space mid-collection (logs can be hundreds of MB across replicas), and the final bundle size is not surfaced to the user. *Suggestion*: fail with an explicit actionable error on write failure (exit 1 — archiving failed), always attempt staging-dir cleanup, and log the final archive size next to its path so users know what they're about to send to support. + +### 3e. Success Measurement + +SC-001 (5 min, single command) and SC-003/004/005 are directly verifiable in e2e; SC-002 (zero artifact follow-ups) is a reasonable post-rollout support-thread metric. Rollback is trivial (new binary, no data-path changes). No findings. + +## Engineering Lens Findings + +### 4a. Architecture Soundness + +- Thin entry point + shared-core collector framework + narrow collect-side backend interface is the right shape; it matches both the constitution and the existing test conventions (fake backend seam instead of a mocking framework). +- **E2 💡 (Multi-container pods)**: The plan's log primitive assumes one container per pod. CNPG database pods and Helm-chart pods with sidecars/init containers will make bare `kubectl logs ` fail ("a container name must be specified"), and `restartCount` lives per *container*, not per pod. *Suggestion*: specify per-container handling in the log primitive — enumerate containers from `containerStatuses`, fetch logs per container (or `--all-containers=true`), and derive `Restarted` per container for `--previous` fetching. + +### 4b. Failure Mode Analysis + +- **E1 🎯 (No collector timeout strategy)** — *cross-lens, see X1*: Every diagnostic collector shells out to `docker exec`/`kubectl exec` against a possibly-degraded instance, and the existing `CommandExecutor` has no timeouts (`CombinedOutput` blocks indefinitely). A wedged container that accepts exec but never returns (rabbitmqctl on a partitioned node, redis-cli on a hung cache, prefect CLI waiting on an unreachable API) hangs the entire run — precisely in the degraded scenario this tool exists for. FR-009 makes failures non-fatal but never defines when a hang *becomes* a failure. *Suggestion (must-address)*: define a per-collector timeout in the plan — run each collector's subprocess under `context.WithTimeout` (e.g. `exec.CommandContext`), default on the order of 60s for exec/status dumps and a larger bound for log downloads, with timeout expiry recorded as `failed` / "timed out after Xs" in the manifest. A single `--collector-timeout`-style knob is optional; a hardcoded sane default is acceptable for v1 if recorded in the manifest reason. +- Partial-failure semantics, interrupt cleanup, and the no-stop guarantee are otherwise well covered. + +### 4c. Security & Privacy Review + +- Key-name masking is an explicit, product-approved parity decision, and its limits are documented to users (review-before-sharing warning). Least privilege is respected (`pods/log` + `pods/exec` read paths only). +- **E3 💡 (Masking application points under-specified)**: research R5 says masking applies to "env output and config dumps", but only Redis `CONFIG GET` and server `env` are named. `rabbitmqctl environment` and `rabbitmqctl status` can also expose credentials-bearing configuration, and future collectors could silently skip masking. *Suggestion*: enumerate the masked outputs normatively (server env vars, server API config dump, Redis `CONFIG GET`, RabbitMQ `environment`/`status`) in the plan/data-model, and route all of them through the single masking function so the list is enforced in one place. + +### 4d. Performance & Scalability + +Sequential collection with a 100k-line default cap across ~8 services × N replicas fits comfortably in SC-001's 5-minute budget (dominated by log transfer); `runCommandPipe` streams to disk without buffering. No caching or async complexity warranted. No findings beyond E1's timeout bound. + +### 4e. Testing Strategy + +- Table-driven unit tests for masking/manifest/outcomes plus fake-backend collector tests match Principle IV and repo convention; e2e scenarios (including degraded-service and multi-replica/previous-log cases) are specified in quickstart. +- **E4 💡 (CI wiring for collect e2e)**: The plan adds `test_docker_collect.py` / `test_k8s_collect.py` but never says which CI jobs run them. The existing e2e workflows are backup-oriented. *Suggestion*: state in the plan (and carry into tasks) that the collect e2e tests join the existing docker/k8s e2e CI jobs (same markers, same fixtures), so they gate merges per Principle IV. + +### 4f. Operational Readiness + +Per-collector streamed progress (format published in docs), explicit exit codes, trivial rollback (delete binary), no migrations. Sufficient for a client-side CLI. No findings. + +### 4g. Dependencies & Integration Risks + +Zero new Go dependencies; docker/kubectl shell-out matches existing backends; Prefect/Redis/RabbitMQ CLI presence inside official images is an explicit spec assumption (same one the Python tool already relies on). `kubectl top` degradation (no metrics-server) handled as a manifest failure. Benchmark image reference deferred to implementation with an env-var override (see Q1). No blocking findings. + +## Cross-Lens Insights + +- **X1 (= E1) 🎯**: The tool's whole product promise is "works when the instance is broken". A hang-forever failure mode is therefore not an edge case but the headline scenario — the engineering fix (timeouts) is also the product fix. Highest-priority item. +- The docs-first approach (contract already published) worked in this feature's favor: both lenses could validate against a concrete, user-visible surface instead of guessing. + +## Findings Summary Table + +| ID | Lens | Severity | Category | Finding | Suggestion | +|----|------|----------|----------|---------|------------| +| E1/X1 | Both | 🎯 | Failure Modes | No timeout strategy: a hung exec on a degraded instance blocks the whole run — the primary use case | Per-collector `context.WithTimeout` subprocess execution; expiry → manifest `failed` with "timed out" reason; defaults defined in plan | +| E2 | Engineering | 💡 | Architecture | Log primitive assumes single-container pods; CNPG/sidecar pods break `kubectl logs` and per-pod restart detection | Enumerate containers per pod; fetch logs and detect restarts per container | +| E3 | Engineering | 💡 | Security | Masked outputs not exhaustively enumerated; RabbitMQ environment/status dumps could ship unmasked | Normative list of masked dumps, all routed through the single masking function | +| E4 | Engineering | 💡 | Testing | Collect e2e tests not wired into CI in the plan | Add collect tests to existing docker/k8s e2e CI jobs | +| P1 | Product | 💡 | Edge Cases/UX | Disk-full behavior unspecified; bundle size not surfaced | Explicit write-failure error + cleanup; log final archive size with path | +| P2 | Product | 💡 | UX | Docker replica log files named by opaque container ID | Use container names on Docker, pod names on Kubernetes | +| Q1 | Both | 🤔 | Dependencies | Benchmark image default reference unknown until ported from the Python tool | Accept: resolve during implementation (tasks item); env-var override already planned | + +## Verdict + +⚠️ **PROCEED WITH UPDATES** — one must-address item (E1), all findings resolvable with bounded edits to `plan.md`/`research.md`/`data-model.md`/contracts. No spec changes required (the spec's requirements are compatible with every fix; FR-009/SC-001 actually *demand* E1's fix). After applying updates, proceed to `/speckit-tasks`. + +**Resolution of Q1 (autonomous)**: accepted as an implementation-time lookup — the image reference is ported from the Python tool's benchmark step, with `INFRAHUB_BENCHMARK_IMAGE` as the override; carried into tasks as an explicit step. No stakeholder input needed beyond the already-confirmed opt-in decision. diff --git a/specs/archive/003-collect-tool/data-model.md b/specs/archive/003-collect-tool/data-model.md new file mode 100644 index 0000000..dc060e2 --- /dev/null +++ b/specs/archive/003-collect-tool/data-model.md @@ -0,0 +1,82 @@ +# Data Model: Infrahub Collect + +**Feature**: 003-collect-tool | **Date**: 2026-07-03 + +All types live in `src/internal/app`. JSON field names below are normative (they appear in published docs); Go identifiers are indicative. + +## BundleManifest + +The collect-side sibling of `BackupMetadata`. Serialized as `bundle_information.json` at the bundle root (`bundle/bundle_information.json` inside the archive). + +| Field (Go) | JSON tag | Type | Rules | +|------------|----------|------|-------| +| ManifestVersion | `manifest_version` | int | Const `manifestVersion = 2026070200` (date-serial, bump on breaking manifest change) | +| CollectID | `collect_id` | string | Timestamp `YYYYMMDD_HHMMSS` (UTC), same ID used in the archive filename | +| CreatedAt | `created_at` | string | RFC3339 UTC (`time.Now().UTC().Format(time.RFC3339)`) | +| ToolVersion | `tool_version` | string | `BuildRevision()` (existing) | +| InfrahubVersion | `infrahub_version` | string | From existing `getInfrahubVersion()`; empty string allowed when server unreachable (recorded as a collector failure, not a run failure) | +| Environment | `environment` | string | `docker` \| `kubernetes` (from detected backend `Name()`) | +| LogLines | `log_lines` | int | Effective per-container log cap for this run (FR-011) | +| Collectors | `collectors` | []CollectorResult | One entry per **attempted or skipped** collector; MUST account for every collector in the run plan (SC-005) | + +**Validation rules**: manifest is written even when collectors fail; it is the last file staged before archiving so it reflects final outcomes. `Collectors` is never empty. + +## CollectorResult + +| Field (Go) | JSON tag | Type | Rules | +|------------|----------|------|-------| +| Name | `name` | string | Stable identifier, path-like for per-service collectors (e.g. `logs/infrahub-server`, `cache-status`, `benchmark`) | +| Status | `status` | string | `success` \| `failed` \| `skipped` (typed constants) | +| Reason | `reason,omitempty` | string | Required when status ≠ `success`; human-readable cause ("container not running", "not requested", "timed out after 60s", "previous logs unavailable") | + +**State transitions**: planned → running → exactly one of `success` / `failed` / `skipped`. A collector never aborts the run (FR-009); the orchestrator converts its returned error into `failed` + reason. + +## Collector + +Internal unit of collection (not serialized). + +| Field | Type | Rules | +|-------|------|-------| +| Name | string | Manifest identifier (see CollectorResult.Name) | +| Run | `func(ctx *collectContext) error` | Writes files under the staging bundle dir; returns wrapped error on failure | +| Skip | optional precondition | Returns (skip bool, reason string) — e.g. benchmark not requested, optional service absent | + +The run plan is an ordered slice built from `CollectOptions` + the canonical service list (`infrahub-server`, `task-worker`, `database`, `message-queue`, `cache`, `task-manager`, `task-manager-db`, `task-manager-background-svc` — the constitution's public deployment contract). + +## CollectOptions + +Input aggregate resolved from flags/env by the entry point and passed to `CollectBundle`. + +| Field | Type | Default | Source | +|-------|------|---------|--------| +| OutputDir | string | `./infrahub_bundles` | `--output-dir` / `INFRAHUB_OUTPUT_DIR` | +| LogLines | int | 100000 | `--log-lines` / `INFRAHUB_LOG_LINES` | +| IncludeBackup | bool | false | `--include-backup` / `INFRAHUB_INCLUDE_BACKUP` | +| IncludeQueries | bool | false | `--include-queries` / `INFRAHUB_INCLUDE_QUERIES` | +| Benchmark | bool | false | `--benchmark` / `INFRAHUB_BENCHMARK` | + +Environment/project selection reuses the existing `Configuration` fields (`DockerComposeProject`, `K8sNamespace`) — not duplicated here. + +## Replica + +Read-only descriptor returned by the backend replica-enumeration primitive (R3). On Kubernetes there is one Replica per **pod container** (multi-container pods — CNPG, sidecars — yield one entry per container; critique E2). + +| Field | Type | Docker meaning | Kubernetes meaning | +|-------|------|----------------|--------------------| +| Service | string | Compose service name | Service name (label-selector resolved) | +| Pod | string | — (empty) | Pod name | +| Container | string | Container **name** (human-meaningful; critique P2) | Container name within the pod | +| Restarted | bool | always false | this container's `restartCount > 0` → previous logs are fetched | + +Log filename derivation: Docker → `.log`; Kubernetes single-container pod → `.log`; multi-container pod → `_.log`; previous logs append `.previous.log` in place of `.log`. + +## Support bundle (filesystem artifact) + +- **Archive**: `/support_bundle_.tar.gz`; all members under a top-level `bundle/` directory; layout identical across environments (FR-006). Normative tree in [contracts/bundle-layout.md](./contracts/bundle-layout.md). +- **Staging**: `os.MkdirTemp(OutputDir, ...)`, removed on success, failure, and interrupt. + +## Relationships to existing entities + +- `InfrahubOps` — orchestrator entry point hangs off it (`iops.CollectBundle(opts)`), reusing `ensureBackend`, `Exec*`, `CopyFrom`, `GetAllPods`, `getInfrahubVersion`, `infrahubInternalAddress`. +- `EnvironmentBackend` — extended with replica/log/metrics read-only primitives; collectors consume a narrow collect-side interface satisfied by both backends (and by the test fake). +- `BackupMetadata` / `CreateBackup` — referenced, unmodified, by the `--include-backup` collector; the backup remains a standard standalone artifact recorded in the manifest. diff --git a/specs/archive/003-collect-tool/opsmill-implement-report.md b/specs/archive/003-collect-tool/opsmill-implement-report.md new file mode 100644 index 0000000..d246637 --- /dev/null +++ b/specs/archive/003-collect-tool/opsmill-implement-report.md @@ -0,0 +1,96 @@ +# Implementation Report: Infrahub Collect (003-collect-tool) + +**Status**: DONE + +**Feature**: Infrahub Collect — third CLI binary `infrahub-collect` for read-only troubleshooting-bundle collection from Docker Compose and Kubernetes Infrahub deployments (Jira INFP-415). + +**Spec dir**: `specs/003-collect-tool` + +**Base commit**: `b1e81e4` (tasks.md committed) + +**Head commit**: `452b10e` + +**Branch**: `fac/collect-tool-implem-opip6` + +All 40 tasks in `tasks.md` are complete (`[X]`), all quality gates pass (`make fmt` / `vet` / `lint` (0 issues) / `test`), and both the Docker and Kubernetes end-to-end suites were observed passing locally. A four-lens review ran across the full diff; all HIGH and MEDIUM findings were fixed inline and re-verified. + +## Chunk-by-chunk ledger + +| # | Chunk (tasks) | Outcome | Commit(s) | Notes | +|---|---------------|---------|-----------|-------| +| 1 | Setup — binary skeleton & build wiring (T001–T004) | 4 ✅ | `9d66dd6` | Flagged: shared `ConfigureRootCommand` surfaces backup-oriented flags on collect's help (pre-existing, same as taskmanager; not in scope to refactor). | +| 2 | Foundational — framework, manifest, masking, orchestrator (T005–T013) | 9 ✅ | `8b89365` | Backend seam resolved via runtime type-assertion (compile-time `var _ collectBackend` assertions added later). Flagged the masking token list missed `requirepass`/`default_pass` → addressed in chunk 4. | +| 3 | US1 — k8s log primitives & service-log collector (T014–T017, T026) | 5 ✅ | `8ccbff4` | Per-container replica model (CNPG/sidecar-safe). | +| 4 | US1 — parity diagnostics, metrics, run plan (T018–T025) | 8 ✅ | `1c4083c` | Widened masking to `pass` substring (subsumes `password`; catches `requirepass`/`default_pass`); added `maskJSON` for the server API config dump. Live smoke run confirmed masking end-to-end. | +| — | Bug fix surfaced by the US1 e2e | ✅ | `e57ef94` | `GetAllPods` lacked the pod-name substring fallback `getPodForService` has, so `infrahub-server` logs were silently skipped under the official Helm chart labels (`infrahub/service=server`). Fixed before the e2e went green. | +| 5 | US1 — k8s e2e (T027–T028) | 2 ✅ | `e79d26e` | Passed locally against a real vcluster+Helm Infrahub 1.10.2 stack (111s): multi-replica logs, induced-restart `.previous.log`, SC-003 before/after read-only snapshot. | +| 6 | US2 — Docker primitives & docker e2e (T029–T032) | 4 ✅ | `df5ea15` | Discovered Infrahub services log to **stderr** → added a combined stdout+stderr pipe primitive (plain stdout would have produced near-empty logs). Stopped services report `failed`, not vanish. 4 docker e2e tests passed locally. | +| 7 | US3 — `--include-backup` (T033–T035) | 3 ✅ | `5ac844c` | Delegates to `CreateBackup` unmodified with `force=false` (preserves the Principle II running-task safety wait); added a documented optional `artifact` manifest field. Full docker e2e re-ran 5/5. | +| 8 | Opt-in benchmark (T036) | 1 ✅ | `86b7bde` | Ported the real image `registry.opsmill.io/opsmill/bench` from upstream `tasks/container_ops.py` (verified via `docker manifest inspect`), `INFRAHUB_BENCHMARK_IMAGE` override. Success + pull-failure (→ `skipped`) paths verified live. | +| 9 | Polish — AGENTS.md, CI wiring, docs (T037–T040) | 4 ✅ | `a78a82d`, `bf60289`, `f8bf345` | AGENTS.md updated to three-binary architecture (constitution follow-up); CI e2e jobs already select collect tests by marker (only relabeled the misleading build step); Go-version doc fix; corrected the benchmark "generates load" wording (it measures host disk/RAM/CPU). | + +## Tasks not completed + +None. All 40 tasks are `[X]` in `tasks.md`. + +## Local-pass evidence + +The implementation added **69 Go unit test functions** (89 including subtests, all passing) plus two Python e2e suites. Full unit evidence lives in the chunk transcripts; representative and safety-critical tests below (all run on Linux 6.8.0-94, Go 1.26.4, `-tags untested_go_version`, worktree `fac/collect-tool-implem-opip6`): + +| Test id | Type | Run command | Passed at (ISO 8601) | Environment context | Verbatim pass line | +|---------|------|-------------|----------------------|---------------------|--------------------| +| `TestRunCollectPlan_OutcomeRecording` | unit | `go test -tags untested_go_version ./src/internal/app/` | 2026-07-03T17:31:25Z | n/a | `--- PASS: TestRunCollectPlan_OutcomeRecording (0.10s)` | +| `TestRunCollectPlan_ReadOnlyGuard` | unit | `make test` | 2026-07-03T20:xx (chunk-fix) | n/a | `--- PASS: TestRunCollectPlan_ReadOnlyGuard (0.00s)` | +| `TestRunCollectPlan_CollectorPanicRecovered` | unit | `make test` | 2026-07-03T20:xx | n/a | `--- PASS: TestRunCollectPlan_CollectorPanicRecovered (0.00s)` | +| `TestCollectDatabaseLogs_IncludeQueries` | unit | `make test` | 2026-07-03T20:xx | n/a | `--- PASS: TestCollectDatabaseLogs_IncludeQueries (0.00s)` | +| `TestAggregatingCollector_TimeoutReasonSurvivesAggregation` | unit | `make test` | 2026-07-03T20:xx | n/a | `--- PASS: TestAggregatingCollector_TimeoutReasonSurvivesAggregation (0.00s)` | +| `TestGetAllPodsWith_Branches` | unit | `make test` | 2026-07-03T20:xx | n/a | `--- PASS: TestGetAllPodsWith_Branches (0.00s)` | +| `TestIsSensitiveKey` / `TestMaskEnvOutput` / `TestMaskErlangConfig` / `TestMaskJSON` | unit | `go test -tags untested_go_version ./src/internal/app/` | 2026-07-03T17:31:25Z | n/a | `--- PASS: TestMaskJSON (0.00s)` | +| `TestBundleManifest_JSONShape` / `_Write` | unit | `make test` | 2026-07-03T17:31:25Z | n/a | `--- PASS: TestBundleManifest_JSONShape (0.00s)` | +| `TestReplicaLogFilename` / `TestParsePodContainerStatuses` | unit | `go test -tags untested_go_version ./src/internal/app/` | 2026-07-03T17:41:15Z | n/a | `--- PASS: TestReplicaLogFilename (0.00s)` | +| `TestParseComposePSContainers` / `TestRunCommandCombinedPipeContext_MergesStderr` | unit | `go test -tags untested_go_version ./src/internal/app/` | 2026-07-03T18:29:20Z | n/a | `--- PASS: TestParseComposePSContainers (0.00s)` | +| `TestIncludeBackupCollector_OutcomeRecording` | unit | `go test -tags untested_go_version ./src/internal/app/` | 2026-07-03T18:51:33Z | n/a | `--- PASS: TestIncludeBackupCollector_OutcomeRecording (0.00s)` | +| `TestBenchmarkCollector_OutcomeRecording` / `TestResolveBenchmarkImage` | unit | `go test -tags untested_go_version ./src/internal/app/` | 2026-07-03T19:17:27Z | n/a | `--- PASS: TestBenchmarkCollector_OutcomeRecording (0.00s)` | +| `tests/e2e/test_k8s_collect.py::test_collect_k8s_bundle` | e2e | `uv run pytest -v -m k8s tests/e2e/test_k8s_collect.py` | 2026-07-03T18:17:52Z | vcluster 0.33.0, chart `oci://registry.opsmill.io/opsmill/chart/infrahub` 4.29.4 (Infrahub 1.10.2, community); cluster torn down by fixture | `tests/e2e/test_k8s_collect.py::test_collect_k8s_bundle PASSED` / `1 passed, 1 warning in 111.09s` | +| `tests/e2e/test_docker_collect.py` (5 tests: full_bundle, project_selection, log_lines, degraded_cache, include_backup) | e2e | `uv run pytest -v -m docker tests/e2e/test_docker_collect.py` | 2026-07-03T20:31:16Z (post-fix re-run) | fixture compose project `infrahub-test-`, image `registry.opsmill.io/opsmill/infrahub:1.8.2`, task-worker scaled to 2, docker 29.2.1 | `=================== 5 passed, 1 warning in 219.12s ===================` | + +Whole-suite confirmation (orchestrator, post-fix): `make test` → `ok infrahub-ops/src/internal/app 0.478s`; 89 `--- PASS` lines, zero failures/skips. + +## Review findings + +A four-lens review (correctness/guidelines, error handling, test coverage, types/comments/simplify) ran across `b1e81e4..HEAD`. The types/comments/simplify lens was performed by the orchestrator inline after two subagent spawn failures. Findings and disposition: + +| Severity | Lens | File | Finding | Disposition | +|----------|------|------|---------|-------------| +| HIGH | correctness | `collect_diagnostics.go` (via both backends' `CopyFrom`) | Neo4j-log copy used context-free `runCommand` → could hang the run forever on a wedged container (worse with `--include-queries`), defeating the "never hang" contract. | **Fixed** `4e91c27` — added `contextCopier`/`CopyFromContext` (bounded by `collectTransferTimeout`) on both backends; shared `CopyFrom` untouched for backup. | +| HIGH | errors | `environment_kubernetes.go` | `ServiceReplicas` mapped *every* `GetAllPods` error (incl. API/RBAC/cluster failure) to "service not deployed"/skipped → exit-0 empty bundle that looks intentional. | **Fixed** `4e91c27` — `errNoPodsMatched` sentinel; genuine empty match → skipped, real kubectl failure → recorded `failed`. | +| HIGH | tests | (missing) | FR-004 `--include-queries` (customer-data-sensitive path) had zero test coverage. | **Fixed** `452b10e` — `TestCollectDatabaseLogs_IncludeQueries` (default / enumerated / empty-fallback). | +| HIGH | tests | (missing) | FR-010 read-only had no cheap unit guard (only the env-gated k8s e2e). | **Fixed** `452b10e` — `TestRunCollectPlan_ReadOnlyGuard` drives the full plan and fails on any Start/Stop/scale. | +| MEDIUM | errors/correctness | `collect.go` loop | No `recover()` around collectors → a panic aborts the whole run (violates FR-009). | **Fixed** `4e91c27` — `safeRun`/`safeSkip` convert panics to a recorded `failed`. | +| MEDIUM | errors/correctness | aggregating collectors | Timeout `*timeoutError` lost through `%v`/`%s` string aggregation → manifest reason was a verbose composite, not the contract's bare `timed out after `; the "covering" test bypassed the real path. | **Fixed** `4e91c27`/`452b10e` — timeout `%w`-wrapped through aggregation; new `TestAggregatingCollector_TimeoutReasonSurvivesAggregation` exercises the real path. | +| MEDIUM | correctness | `collect_diagnostics.go`, `app.go` | server-info version/internal-address lookups + k8s pod resolution on the collect path were unbounded. | **Fixed** `4e91c27` — bounded collect-local variants; shared helpers untouched for backup. | +| LOW | correctness | `collect_logs.go` | Docker combined-pipe reader not closed (fd leak). | **Fixed** `4e91c27` — `defer reader.Close()`. | +| LOW | errors | `collect_diagnostics.go` | `--include-queries` empty enumeration → silent empty `database/` success. | **Fixed** `4e91c27` — falls back to `neo4j.log`+`debug.log`. | +| LOW | errors | `collect_diagnostics.go` | `events.json` could contain a Python traceback on failure. | **Fixed** `4e91c27` — written only on success; failure output → `events.err.txt`. | +| LOW | errors | `collect_prefect_events.py` | Pagination could spin on a truthy `next_page` with zero events. | **Fixed** `4e91c27` — max-pages cap + break on empty page. | +| LOW-MED | tests | `collect_manifest_test.go` | Brittle `len==8` field-count assertion contradicts the schema's `additionalProperties: true`. | **Fixed** `452b10e` — asserts required-field presence instead. | +| LOW | types (retracted) | both backends | Suspected missing compile-time `var _ collectBackend` assertions. | **Not a defect** — assertions exist in a grouped `var(...)` block (initial grep false-negative). Retracted after verification. | +| LOW / by-design | correctness | `masking.go` | Key-name masking misses connection-string credentials whose key lacks pass/secret/token/key (e.g. `..._URL=postgresql://user:pw@…`); multi-line env values mask only the first line. | **Deferred** — explicit FR-008 parity-with-Python decision; user docs already warn to review bundles before sharing. | +| LOW | correctness | `utils.go` `createTarball` | A second SIGINT mid-archive-write could leave a truncated file under the final name (no temp-then-rename). | **Deferred** — very narrow window; single SIGINT is handled cleanly. | +| LOW / by-design | correctness | `backup_containers.go` (via `--include-backup`) | Inherited `waitForRunningTasks` can block on a busy instance. | **Deferred** — documented delegated backup behavior (R10), outside collect's own contract. | + +## Autonomous decisions + +- **Masking token list widened to `pass`** (chunk 4): FR-008 specifies the list as a *minimum* ("at least keys containing password/secret/token/key"), so adding `pass` to catch `requirepass`/`default_pass` is in-spec; over-masking is safe. `maskJSON` was added for the server API config dump named in research R5. +- **`--include-backup` uses `force=false`** (chunk 7): preserves the backup tool's running-task safety wait (Principle II) at the cost of a possible long wait on a busy instance — safety was chosen over bounded runtime. The unbounded wait is recorded as a deferred LOW review item. +- **Benchmark image ported, not guessed** (chunk 8): fetched the real reference from upstream `opsmill/infrahub` `tasks/container_ops.py` and verified it exists; the docs' "generates load" claim was corrected to reflect that it measures host requirements. +- **Types/comments/simplify review done inline** (Phase 6): the review subagent failed to run twice (0 tool uses, boilerplate output containing a spurious "System:"-style line, which was disregarded as data, not instruction); rather than retry a third time, the orchestrator performed that lens directly and verified each finding against the code — which caught and retracted one false-positive. +- **Review scope** was `b1e81e4..HEAD` (this run), not the broader `main...HEAD` the detect script reports — earlier branch commits (backup docs) predate this run and were out of scope. +- **k8s e2e not re-run after the fixes**: the fixes' Kubernetes logic (FIX-2 sentinel, FIX-5 bounded resolution) is covered by new unit tests (`TestGetAllPodsWith_Branches`, pod-resolution tests); the docker e2e was re-run 5/5 green because the fixes touched shared diagnostics/backend code. + +## Suggested next steps + +1. **Open a PR** for `fac/collect-tool-implem-opip6` → `main`. CI will run the Docker and Kubernetes e2e suites (both already carry the right pytest markers, per T038). +2. **Watch the first CI run's wall-clock**: the collect e2e adds ~2–6 min per suite; the existing 30-min job budget was left unchanged (flagged in chunk 9) — bump it if CI comes close. +3. **Optional follow-ups** for the deferred LOW items if desired: temp-then-rename in `createTarball` for the double-SIGINT window, and a docs note about connection-string credentials not being masked (the guide already warns generally). +4. Run `/speckit.opsmill.extract` to capture reusable knowledge/ADRs from this feature (the manual follow-up step this pipeline intentionally leaves to you). diff --git a/specs/archive/003-collect-tool/plan.md b/specs/archive/003-collect-tool/plan.md new file mode 100644 index 0000000..3650419 --- /dev/null +++ b/specs/archive/003-collect-tool/plan.md @@ -0,0 +1,111 @@ +# Implementation Plan: Infrahub Collect (Troubleshooting Bundle Tool) + +**Branch**: `fac/collect-tool-implem-opip6` (feature `003-collect-tool`) | **Date**: 2026-07-03 | **Spec**: [spec.md](./spec.md) + +**Input**: Feature specification from `/specs/003-collect-tool/spec.md` + +## Summary + +Add a third CLI binary `infrahub-collect` whose `create` command gathers a troubleshooting bundle — per-service logs (all replicas, plus previous-container logs on Kubernetes), parity diagnostics (Neo4j, RabbitMQ, Redis, Prefect worker/manager, server info), and container metrics — into a single local `support_bundle_.tar.gz` with a `bundle_information.json` manifest recording an explicit outcome per collector. + +Technical approach: a thin Cobra entry point at `src/cmd/infrahub-collect` mirroring the two existing binaries, with all logic in `src/internal/app`. Collection reuses the existing `EnvironmentBackend` abstraction (Docker Compose / Kubernetes detection, exec, copy, CNPG primary targeting) and `CommandExecutor`, extended with net-new read-only primitives for replica enumeration, log streaming, and resource metrics. A new collector framework runs each collector independently (failures are non-fatal), masks secrets in env/config output by key name, stages files under a working directory, and packages them with the existing `createTarball` helper. `--include-backup` delegates to the existing `CreateBackup` unmodified; `--benchmark` is opt-in and degrades to skipped when its image is unavailable. The user-facing contract (commands, flags, env vars, bundle layout, manifest fields) is already published in `docs/docs/` and is treated as normative. + +## Technical Context + +**Language/Version**: Go 1.25.0 (`CGO_ENABLED=0`, `-tags untested_go_version`; cross-compiles linux/darwin/windows × amd64/arm64 via `make build-all`) + +**Primary Dependencies**: cobra v1.10.2, viper v1.21.0, logrus v1.9.4 (all existing). Docker and Kubernetes interaction shells out to the `docker` and `kubectl` CLIs through the existing `CommandExecutor` — no client-go or Docker SDK. **No new Go module dependencies expected**; if any are added, `scripts/update-vendor-hash.sh` must be run (constitution, Operational Constraints). + +**Storage**: local filesystem only — bundle staging directory and final `.tar.gz` under `--output-dir` (default `./infrahub_bundles`, env `INFRAHUB_OUTPUT_DIR`). No network egress beyond the target deployment (FR-012); S3 upload explicitly out of scope (INFP-581). + +**Testing**: `go test` table-driven unit tests in `src/internal/app/*_test.go` for pure logic (masking, manifest construction, collector outcome recording, replica/log-arg building) using a fake backend; Python/pytest end-to-end tests in `tests/e2e/` (`-m docker` / `-m k8s` markers) invoking the built binary as a subprocess, following the existing `backup_binary` fixture pattern. + +**Target Platform**: operator workstations and CI runners (Linux/macOS/Windows) with Docker CLI or kubectl access to the target deployment; runs outside the cluster. + +**Project Type**: CLI — third binary in an established split-binary, shared-core Go repository. + +**Performance Goals**: complete a bundle from a reference deployment in under 5 minutes with a single command (SC-001); stream progress per collector in real time (Principle V). + +**Constraints**: strictly read-only with respect to workload lifecycle — no stop/restart/scale ever (FR-010, SC-003); fully offline by default — no image pulls, no egress (FR-012); individual collector failures are non-fatal and the command still exits 0 with a partial bundle (FR-009, SC-005); every collector subprocess is time-bounded (`exec.CommandContext`; 60s exec dumps, 5 min log/copy transfers — critique E1) so a hung container on a degraded instance cannot stall the run; identical command surface and bundle layout across Docker and Kubernetes (FR-006, SC-004); per-service log cap configurable, default 100,000 lines (FR-011). + +**Scale/Scope**: 8 Infrahub services × N replicas per service (Kubernetes); log volume up to `--log-lines` per container (default 100k lines); one archive per run. + +## Constitution Check + +*GATE: evaluated against constitution v1.1.0 — pre-Phase-0 PASS, re-checked post-Phase-1 PASS.* + +| Principle | Verdict | Evidence | +|-----------|---------|----------| +| I. Split-Binary, Shared-Core | ✅ PASS | Constitution v1.1.0 already enumerates `src/cmd/infrahub-collect` (amendment ratified 2026-07-02). Entry point is thin Cobra wiring only; all collection logic lands in `src/internal/app`. Mandated follow-up: AGENTS.md/CLAUDE.md still say "two specialized CLI binaries" and MUST be updated in this feature (constitution Sync Impact Report). | +| II. Operational Safety & Data Integrity | ✅ PASS | Collection is stricter than backup: read-only, never stops/starts workloads (FR-010). No destructive operations, so no confirmation flags required. `--include-backup` delegates to the existing `CreateBackup` path unmodified, preserving its stop/restart-on-failure and metadata/checksum guarantees. Bundle manifest is the collect-side analog of backup metadata (FR-007). | +| III. Self-Contained, Cross-Platform Binaries | ✅ PASS | No new runtime assets beyond possibly small embedded scripts via the existing `go:embed scripts/*` pattern. Depends only on docker/kubectl already present in the target environment; absence fails fast with the existing `ErrCLIUnavailable` actionable error (FR requirement, edge case "Missing CLI prerequisites"). `CGO_ENABLED=0` build via existing Makefile matrix. | +| IV. Quality Gates: Test, Lint, Vet | ✅ PASS | Plan includes table-driven unit tests for all new pure logic (masking, manifest, collector outcomes) and new pytest e2e collect tests for Docker and Kubernetes flows. `make test`, `make vet`, `make lint`, `make fmt` gate merge as usual. | +| V. Explicit Errors & Streaming Feedback | ✅ PASS | Collectors wrap errors with `fmt.Errorf`/`%w` and return them to the orchestrator, which records them in the manifest and logs warnings; no `os.Exit` in `src/internal/app`. Per-collector progress streams via logrus as each collector runs (docs show the exact expected output). | +| Operational Constraints | ✅ PASS | Docker Compose service names / K8s label selectors reused as-is (public deployment contract, unchanged). No change to backup snapshot layout. Docs already exist and passed Vale/rumdl in prior commits. `go.mod` unchanged → no vendor-hash update expected. | + +**Gate result**: PASS — no violations, Complexity Tracking not required. + +## Project Structure + +### Documentation (this feature) + +```text +specs/003-collect-tool/ +├── spec.md # Feature specification (complete) +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ # Phase 1 output +│ ├── cli.md # Command/flag/env/exit-code contract (normative) +│ ├── bundle-layout.md # Archive layout contract +│ └── manifest.schema.json # bundle_information.json JSON Schema +├── checklists/requirements.md # Spec quality checklist (complete) +└── tasks.md # Phase 2 output (/speckit-tasks — NOT created by plan) +``` + +### Source Code (repository root) + +```text +src/ +├── cmd/ +│ ├── infrahub-backup/main.go # unchanged +│ ├── infrahub-taskmanager/main.go # unchanged +│ └── infrahub-collect/main.go # NEW: thin Cobra wiring — root cmd, create cmd, +│ # flags (--output-dir, --log-lines, --include-backup, +│ # --include-queries, --benchmark), version cmd, +│ # ConfigureRootCommand + AttachEnvironmentCommands +└── internal/app/ + ├── collect.go # NEW: CollectBundle orchestrator — staging dir, collector + │ # loop (non-fatal failures), archive, cleanup + ├── collect_manifest.go # NEW: BundleManifest, CollectorResult, status constants, + │ # manifest construction + JSON writing + ├── collect_logs.go # NEW: service-log collectors (all replicas, previous logs) + ├── collect_diagnostics.go # NEW: parity collectors — database, message-queue, cache, + │ # task-worker, task-manager, server info + ├── collect_metrics.go # NEW: container metrics (docker stats / kubectl top) + ├── collect_extras.go # NEW: opt-in collectors — include-backup, benchmark + ├── masking.go # NEW: key-name secret masking (password|secret|token|key) + ├── environment.go # EXTEND: replica/log/metrics read-only primitives + ├── environment_docker.go # EXTEND: compose ps -q replica listing, docker logs, stats + ├── environment_kubernetes.go # EXTEND: pod replica listing w/ restart counts, + │ # kubectl logs [--previous], kubectl top + ├── cli.go # unchanged (shared flags already cover --project, + │ # --k8s-namespace, --log-format) + └── *_test.go # NEW: masking_test.go, collect_manifest_test.go, + # collect_logs_test.go (fake backend) + +tests/e2e/ +├── test_docker_collect.py # NEW: compose collect e2e (+ degraded-service case) +└── test_k8s_collect.py # NEW: k8s collect e2e (replicas, previous logs) + +Makefile # EXTEND: BINARIES += infrahub-collect +flake.nix # EXTEND: third buildGoModule package + symlinkJoin entry +AGENTS.md # EXTEND: two → three binaries (constitution follow-up) +``` + +**Structure Decision**: single Go project, existing layout. The new binary is a sibling of the two existing entry points; all new logic lives in `src/internal/app` as `collect_*.go` files plus a shared `masking.go`. Backend extensions are added as methods on the existing `DockerBackend`/`KubernetesBackend` types and surfaced to collectors through a narrow collect-side interface so collectors are unit-testable with a fake. The Dockerfile and Homebrew/Docker release jobs remain `infrahub-backup`-only in v1 (no collect container image is promised in docs; the S3 binary upload in `release.yml` picks up the new binary automatically through `BINARIES`). + +## Complexity Tracking + +No constitution violations — table not required. diff --git a/specs/archive/003-collect-tool/quickstart.md b/specs/archive/003-collect-tool/quickstart.md new file mode 100644 index 0000000..bded3e6 --- /dev/null +++ b/specs/archive/003-collect-tool/quickstart.md @@ -0,0 +1,95 @@ +# Quickstart Validation: Infrahub Collect + +**Feature**: 003-collect-tool | Validation scenarios proving the feature end-to-end. Contracts: [cli.md](./contracts/cli.md), [bundle-layout.md](./contracts/bundle-layout.md), [manifest.schema.json](./contracts/manifest.schema.json). + +## Prerequisites + +- Go 1.25 toolchain (`make dev-setup` for lint tooling) +- Docker with Docker Compose (Docker scenarios) +- kind + kubectl + helm (Kubernetes scenarios; e2e fixtures under `tests/e2e/fixtures/helm/`) +- Python + pytest for e2e (`tests/e2e/`) + +## 1. Build and static gates + +```bash +make build # bin/infrahub-backup, bin/infrahub-taskmanager, bin/infrahub-collect +make test # unit tests incl. masking / manifest / collector tables +make vet && make lint && make fmt +``` + +**Expected**: all three binaries build; all gates pass. + +## 2. CLI surface smoke test + +```bash +bin/infrahub-collect version # -> "Version: " +bin/infrahub-collect --help # shows create + environment + version +bin/infrahub-collect create --help # shows --log-lines, --include-backup, + # --include-queries, --benchmark +bin/infrahub-collect environment detect +bin/infrahub-collect environment list +``` + +**Expected**: flags/commands match [contracts/cli.md](./contracts/cli.md); environment commands behave identically to `infrahub-backup`. + +## 3. Docker Compose collection + +With an Infrahub compose project running: + +```bash +bin/infrahub-collect create --output-dir=/tmp/bundles +BUNDLE=$(ls -t /tmp/bundles/support_bundle_*.tar.gz | head -1) + +tar -tzf "$BUNDLE" > /dev/null && echo "archive OK" +tar -xzOf "$BUNDLE" bundle/bundle_information.json | jq '.environment, .collectors' +``` + +**Expected**: exit 0; archive valid; manifest reports `"docker"`, one entry per collector, all `success` on a healthy stack; `bundle/logs//` populated for every running service; env dumps in `bundle/server/` contain `********` for keys matching `password|secret|token|key` and no plaintext secrets. + +## 4. Degraded-instance collection (FR-009 / SC-005) + +```bash +docker compose -p stop cache +bin/infrahub-collect create --output-dir=/tmp/bundles; echo "exit=$?" +``` + +**Expected**: `exit=0`; a `WARN` line for the cache collector; manifest entry `{"name": "cache-status", "status": "failed", "reason": ...}`; every other collector still reported. Restart cache afterwards. + +## 5. Kubernetes collection (multi-replica + previous logs) + +Against a kind cluster with the Helm chart installed (task-worker scaled ≥ 2, one pod restarted at least once): + +```bash +bin/infrahub-collect create --k8s-namespace --output-dir=/tmp/bundles +``` + +**Expected**: manifest `environment: "kubernetes"`; `bundle/logs/task-worker/` contains one `.log` per pod and a `.previous.log` for the restarted pod; layout byte-identical in structure to the Docker bundle; **zero** pod restarts/scale events attributable to the run (`kubectl get events`) — FR-010/SC-003. + +## 6. Flag and env-var behavior + +```bash +bin/infrahub-collect create --log-lines=500 --output-dir=/tmp/bundles +INFRAHUB_LOG_LINES=250 bin/infrahub-collect create --output-dir=/tmp/bundles +``` + +**Expected**: manifests record `"log_lines": 500` and `250` respectively; flag wins over env when both are set. + +## 7. Opt-in collectors + +```bash +bin/infrahub-collect create --include-backup # backup artifact created next to bundle, + # referenced in manifest; bundle still + # produced if backup fails +bin/infrahub-collect create --include-queries # bundle/database/ includes query logs +bin/infrahub-collect create --benchmark # benchmark results in bundle/benchmark/; + # air-gapped -> status "skipped" + warning +``` + +## 8. End-to-end suites + +```bash +pytest tests/e2e -m docker -k collect +pytest tests/e2e -m k8s -k collect +``` + +**Expected**: new `test_docker_collect.py` / `test_k8s_collect.py` pass, covering scenarios 3–6 automatically. diff --git a/specs/archive/003-collect-tool/research.md b/specs/archive/003-collect-tool/research.md new file mode 100644 index 0000000..c03bc79 --- /dev/null +++ b/specs/archive/003-collect-tool/research.md @@ -0,0 +1,97 @@ +# Phase 0 Research: Infrahub Collect + +**Feature**: 003-collect-tool | **Date**: 2026-07-03 + +All unknowns from Technical Context were resolved by studying the existing codebase (backends, executor, backup pipeline, build surface), the published user docs (which define the normative CLI/bundle contract), and the spec's parity list. No external research tasks remained open. + +## R1. Kubernetes and Docker interaction: shell out to CLIs, no client libraries + +- **Decision**: All Docker and Kubernetes operations shell out to the `docker` and `kubectl` binaries through the existing `CommandExecutor`, exactly like the current backends. No client-go, no Docker SDK. +- **Rationale**: Consistency with the entire existing codebase (`environment_docker.go`, `environment_kubernetes.go`); zero new dependencies keeps `CGO_ENABLED=0` cross-compilation trivial and avoids a vendor-hash churn; kubectl handles kubeconfig/auth plugins (OIDC, cloud IAM) for free, which client-go would force us to reimplement. +- **Alternatives considered**: `k8s.io/client-go` (rejected: large dependency tree, auth-plugin complexity, inconsistent with existing backends); Docker Engine API via socket (rejected: breaks remote-context workflows that the docker CLI handles transparently). + +## R2. Collector framework: named units with independent, non-fatal outcomes + +- **Decision**: A `collector` is a named unit (`name`, `run(bundleDir) error`, optional skip precondition). The orchestrator (`CollectBundle`) iterates a fixed, ordered collector list; each failure is caught, logged as a warning, recorded in the manifest as `failed` with a reason, and the loop continues. Collectors that don't apply (benchmark not requested, optional service absent) record `skipped` with a reason. The command exits 0 whenever the archive is produced, even with failures (FR-009). +- **Rationale**: Directly encodes FR-007/FR-009/SC-005 (manifest accounts for 100% of attempted collectors with explicit outcomes). Sequential execution keeps output streaming readable (docs show sequential progress lines) and avoids hammering a degraded instance; per-collector wall time is dominated by log download, well within SC-001's 5-minute budget. +- **Timeouts (critique E1, must-address)**: every collector subprocess runs under `context.WithTimeout` via `exec.CommandContext` — a degraded instance can accept an exec and never return (wedged container, partitioned RabbitMQ node, hung kubelet), and without a bound a single hang blocks the entire run, defeating FR-009 and SC-001. Defaults: 60s per exec/status-dump command, 5 minutes per replica log download and for the copy-based Neo4j log collector; timeout expiry is recorded in the manifest as `failed` with reason `timed out after `. Hardcoded defaults for v1 (no new flag); the reason string makes the bound visible to support. +- **Alternatives considered**: Parallel collector execution (rejected for v1: interleaved streaming output, higher load on degraded instances, no evidence the 5-minute budget needs it); error-out on first failure (rejected: contradicts FR-009 — a degraded instance is the primary use case); a `--collector-timeout` flag (deferred: not in the published CLI contract; add later if field experience demands it). + +## R3. Log collection: extend both backends with read-only replica/log primitives + +- **Decision**: Extend the backends with net-new read-only methods, surfaced to collectors via a narrow collect-side interface (defined next to the collectors, satisfied by both backends): + - `ServiceReplicas(service) ([]Replica, error)` — Docker: `docker compose -p ps ` → one `Replica` per container, carrying the **container name** (not the bare ID) so bundle filenames are human-meaningful (critique P2); Kubernetes: existing `GetAllPods` + `kubectl get pod -o jsonpath` over `status.containerStatuses` → one `Replica` per **pod container** (critique E2: CNPG and sidecar-bearing pods are multi-container; bare `kubectl logs ` fails there, and restart counts live per container). `Restarted` derives from that container's `restartCount > 0`. + - `ReplicaLogs(replica, tailLines, previous) (io.ReadCloser, wait, error)` — Docker: `docker logs --tail ` (previous unsupported → never requested); Kubernetes: `kubectl logs -n -c --tail= [--previous]`. Built on the existing `runCommandPipe` streaming primitive so large logs stream to disk without buffering in memory. +- **Rationale**: `GetAllPods` (multi-replica) and `runCommandPipe` (streaming) already exist; only the log commands are new. Per-replica files match the promised bundle layout (`logs//` one file per replica, `*.previous.log` for restarted pods); single-container pods (the normal case) keep plain `.log` names, multi-container pods get `_.log`. Fetching previous logs only when the container's `restartCount > 0` avoids a guaranteed-failing kubectl call per healthy container. +- **Alternatives considered**: `docker compose logs ` (rejected: interleaves replicas into one stream and prefixes lines, breaking per-replica files and parity with k8s layout); `kubectl logs --all-containers=true` (rejected: merges containers into one stream, losing per-container previous-log targeting); always attempting `--previous` and swallowing errors (rejected: noisy, slower, indistinguishable from real failures in the manifest). + +## R4. Container metrics: one-shot stats appropriate to each environment + +- **Decision**: Docker: `docker stats --no-stream ` for the project's containers; Kubernetes: `kubectl top pods -n `. Output captured as plain text into `metrics/`. On Kubernetes without metrics-server, the collector records `failed` with the kubectl error as reason — non-fatal. +- **Rationale**: Matches FR-005 (container stats on Docker, pod resource metrics on Kubernetes; no host metrics) and the docs' collectors table (`docker compose stats` / `kubectl top`). metrics-server absence is a routine cluster condition → belongs in the manifest, not a run failure. +- **Alternatives considered**: cAdvisor/Prometheus scraping (rejected: assumes infrastructure that may not exist, violates offline-by-default simplicity); skipping metrics on k8s without metrics-server silently (rejected: FR-007 requires explicit outcomes). + +## R5. Secret masking: pure key-name matcher applied to env and config dumps + +- **Decision**: A pure function in `masking.go`: case-insensitive substring match on key names — `password`, `secret`, `token`, `key` — replaces the value with `********`. The masked outputs are enumerated normatively (critique E3) and every one of them is routed through this single function before being written to the bundle: (1) `infrahub-server` environment variables (`env` dump), (2) the server API configuration dump, (3) Redis `CONFIG GET '*'`, (4) RabbitMQ `rabbitmqctl environment`, (5) RabbitMQ `rabbitmqctl status`. Raw service logs are written as-is (parity with today's tool; documented in the docs warning). +- **Rationale**: Exact parity with FR-008 and the published docs; a single choke-point function keeps the masked-output list enforceable and prevents future collectors from silently skipping masking. A pure function is trivially table-testable (Principle IV names configuration logic as a unit-test target). +- **Alternatives considered**: Pattern-based scrubbing of secret *values* inside logs (explicitly out of scope for v1 per spec assumptions); allowlist approach (rejected: inverts the current tool's behavior, breaking parity). + +## R6. Bundle manifest: sibling of BackupMetadata, normative field set from docs + +- **Decision**: `BundleManifest` struct in `collect_manifest.go` with JSON tags exactly matching the published example: `manifest_version` (const `2026070200`), `collect_id` (timestamp `YYYYMMDD_HHMMSS`), `created_at` (RFC3339 UTC), `tool_version` (`BuildRevision()`), `infrahub_version` (existing `getInfrahubVersion()`), `environment` (`docker`|`kubernetes`), `log_lines` (effective cap), `collectors` (array of `{name, status, reason?}`, status ∈ `success|failed|skipped`). Written as `bundle_information.json` via `json.MarshalIndent` at the bundle root. +- **Rationale**: Mirrors the proven `BackupMetadata` pattern (`metadataVersion` date-serial const, `MarshalIndent`, `os.WriteFile`) and the docs' manifest example verbatim, which is the user-facing contract. +- **Alternatives considered**: Reusing/extending `BackupMetadata` (rejected: different lifecycle and fields; spec names the manifest as a *sibling*, not an extension). + +## R7. Archive packaging and interrupt safety + +- **Decision**: Stage files in a temporary working directory created inside the output directory (`os.MkdirTemp(outputDir, ...)`), lay files out under `/bundle/...`, then reuse the existing `createTarball(archivePath, workDir, "bundle/")` and remove the staging dir. Cleanup runs via `defer`; a signal handler (SIGINT/SIGTERM) also removes the staging dir so interrupts leave no temp files (edge case "Interrupted collection"). Since collection never stops workloads, no start/restart recovery is needed. Write failures on the staging dir or archive (e.g. disk full — critique P1) are hard errors: exit 1 with an actionable message naming the output directory, after cleanup. On success the final log line reports the archive path **and size**, so users know what they are about to send to support. +- **Rationale**: `createTarball` is exactly the mechanism backups use (`workDir` + path-in-tar prefix); staging inside the output dir keeps the final rename/write on one filesystem and confines all writes to the user-chosen location. +- **Alternatives considered**: Streaming tar writer without a staging dir (rejected: prevents retro-writing the manifest, which must be finalized *after* all collectors report); staging in `/tmp` (rejected: cross-device writes, files leak outside the user-designated directory). + +## R8. Server info collection: reuse internal-address plumbing and exec + +- **Decision**: The `server` collector execs inside `infrahub-server`: Infrahub version (existing `getInfrahubVersion`), installed packages (`pip list`), environment variables (`env`, masked per R5), and API information/configuration/schema fetched from inside the container against the existing `infrahubInternalAddress` (Python one-liners / an embedded script via the existing `script_executor.go` + `go:embed` pattern, since the server image ships Python but not necessarily curl). +- **Rationale**: `infrahubInternalAddress` and the embedded-script execution pipeline already exist and are constitution-compliant (Principle III: assets via `go:embed`). Exec-from-inside avoids any assumption about external API exposure — collection stays workable when the API is not reachable from the operator's workstation. +- **Alternatives considered**: Calling the Infrahub API from the collect binary over the network (rejected: requires reachable ingress + credentials; violates "no network access beyond the deployment" simplicity since exec transport already exists). + +## R9. Parity diagnostics: fixed exec command sets per service + +- **Decision**: Exec-based collectors issue the parity commands inside the official containers, targeting the CNPG/HA primary automatically through the existing `getPodForService`/`findPrimaryPod` behavior: + - `database` (Neo4j): copy `neo4j.log` and `debug.log` via existing `CopyFrom`; with `--include-queries`, copy the full log directory including query logs (FR-004). + - `message-queue` (RabbitMQ): `rabbitmqctl list_queues / list_exchanges / list_bindings / list_connections / list_channels / status / environment`. + - `cache` (Redis): `redis-cli info / client list / config get '*' (masked) / slowlog get / dbsize`. + - `task-worker`: Prefect worker CLI output, one directory per replica (via R3 replica enumeration). + - `task-manager`: Prefect work pools, work queues, recent flow runs, events, automations via Prefect CLI/API inside the container (embedded-script pattern where CLI output is insufficient). +- **Rationale**: This is the FR-003 parity list; the spec's assumption confirms these CLIs exist inside the official images. Absent optional services (`task-manager-background-svc`) record `skipped` (edge case). +- **Alternatives considered**: Talking to RabbitMQ/Redis/Prefect over the network from the workstation (rejected: ports are typically not exposed; exec transport is the established pattern). + +## R10. `--include-backup`: delegate to CreateBackup unmodified + +- **Decision**: When `--include-backup` is set, the orchestrator invokes the existing `CreateBackup` entry point with collect-appropriate defaults (non-interactive, no S3 upload, no redaction/encryption), records the produced backup's identity/path in the manifest, and records `failed` (with reason) while still producing the bundle if backup errors (FR-014, US3 acceptance scenario 2). The backup artifact stays a separate file next to the bundle — it is referenced by, not embedded in, the archive. +- **Rationale**: FR-014 says "reuses the existing backup behavior unmodified"; embedding a multi-GB backup inside the bundle tarball would double disk usage and break the backup tool's own restore discovery. The docs' warning that `--include-backup` may stop containers documents the inherited behavior. +- **Alternatives considered**: Embedding the backup archive inside the bundle (rejected: disk-space doubling, restore tooling expects the standard backup location/naming); re-implementing a lighter read-only backup (rejected: violates "unmodified" and Principle II's guarantees). + +## R11. `--benchmark`: opt-in, image-based, graceful degradation + +- **Decision**: Default off (FR-013). When requested, the collector runs the OpsMill benchmark container against the deployment (Docker: `docker run` attached to the project network; Kubernetes: a one-off `kubectl run` pod in the namespace), captures its output into `benchmark/`, and removes the transient container/pod it created (which is permitted: FR-010 protects *the deployment's* workloads; the benchmark pod is the tool's own transient resource, and `--benchmark` explicitly opts into image download). The image reference is a build-time default overridable via `INFRAHUB_BENCHMARK_IMAGE`; the concrete default is ported from the Python tool's benchmark step during implementation. Pull or run failure → collector `skipped` with a warning in the manifest (air-gapped edge case). +- **Rationale**: Matches FR-013 and the docs (`--benchmark` requires image download, skipped with a warning when unavailable). Override env var keeps air-gapped-with-private-registry workflows possible. +- **Alternatives considered**: Embedding a Go-native benchmark (rejected: duplicates the maintained benchmark suite); making benchmark default-on like the Python tool (rejected: spec decision — offline-by-default won). + +## R12. CLI wiring: flags, env vars, and shared commands + +- **Decision**: `src/cmd/infrahub-collect/main.go` mirrors the taskmanager entry point: `app.SetVersion`, `NewInfrahubOps`, `ConfigureRootCommand` (brings `--project`, `--k8s-namespace`, `--log-format` + `INFRAHUB_*` env binding), `AttachEnvironmentCommands` (brings `environment detect|list`), a `version` command, and a `create` command. Collect-specific flags are registered in main.go and bound to viper: `--output-dir` (persistent, default `./infrahub_bundles`, `INFRAHUB_OUTPUT_DIR`), and on `create`: `--log-lines` (default 100000, `INFRAHUB_LOG_LINES`), `--include-backup`, `--include-queries`, `--benchmark` (each with its `INFRAHUB_*` equivalent per the docs' reference tables). Precedence flag > env > default comes from viper as today. +- **Rationale**: Exactly the published commands reference; flag registration in entry points is the established, constitution-sanctioned pattern (backup's main.go does the same). `--output-dir` is collect-specific, so it does not belong in the shared `ConfigureRootCommand` (backup already has its own `--backup-dir`). +- **Alternatives considered**: Adding `--output-dir` to `ConfigureRootCommand` (rejected: would surface a meaningless flag on the other two binaries). + +## R13. Testing strategy + +- **Decision**: Unit tests (Go, table-driven, no mocking framework — repo convention): masking matrix, manifest construction/serialization against the documented JSON shape, collector outcome recording (success/failed/skipped/timed-out paths) using a fake collect-backend, log-command argument construction, replica parsing. E2E (pytest, existing markers/fixtures): `test_docker_collect.py` — full collect against the compose stack, archive integrity, manifest completeness, per-service logs present, masked env contains no plaintext secrets, degraded case (one service stopped → exit 0 + `failed` entry); `test_k8s_collect.py` — same against the kind+Helm stack plus multi-replica and previous-log assertions. New `collect_binary` fixture and `run_collect` helper mirror the backup ones. The collect e2e tests join the **existing** docker/k8s e2e CI jobs (same pytest markers, same fixtures, same workflow matrix in `.github/workflows/ci.yml`) so they gate merges per Principle IV (critique E4). +- **Rationale**: Principle IV mandates table-driven unit tests for pure logic and e2e jobs for container-touching flows; the fake-backend seam exists naturally because collectors consume a narrow interface (R3). +- **Alternatives considered**: Introducing a mocking framework for `CommandExecutor` (rejected: against repo convention; the interface seam gives the same testability). + +## R14. Build/release surface + +- **Decision**: Add `infrahub-collect` to `BINARIES` in the Makefile (wires `build`, `build-all`, `install`); add a third `buildGoModule` package and symlinkJoin entry in `flake.nix`; `release.yml` uploads `./bin` wholesale so the new binary ships automatically at the documented URL pattern. Dockerfile and the Docker/Homebrew release jobs stay `infrahub-backup`-only (no collect image or formula is promised). Update AGENTS.md (and its CLAUDE.md routing) from "two specialized CLI binaries" to three — the constitution's Sync Impact Report flags this as a mandatory companion change. +- **Rationale**: Minimal, mechanical, matches the install docs (S3 download URL + `make build`). `go.mod` is untouched, so `vendorHash` stays as-is. +- **Alternatives considered**: Multi-binary Docker image (rejected for v1: not in docs, expands release surface without a user story). diff --git a/specs/003-collect-tool/spec.md b/specs/archive/003-collect-tool/spec.md similarity index 99% rename from specs/003-collect-tool/spec.md rename to specs/archive/003-collect-tool/spec.md index 9628976..5a0897a 100644 --- a/specs/003-collect-tool/spec.md +++ b/specs/archive/003-collect-tool/spec.md @@ -4,7 +4,7 @@ **Created**: 2026-07-02 -**Status**: Draft +**Status**: Extracted **Input**: User description: "Add a new third CLI binary `infrahub-collect` (troubleshooting-bundle collection tool) to the infrahub-ops codebase, replacing the Python `invoke bundle collect` script from the main Infrahub repo and adding first-class Kubernetes support. Source: Jira INFP-415 and its Discovery Brief; upload is explicitly out of scope (deferred to INFP-581)." (Full grilled idea brief provided in conversation; confirmed decisions: third binary, parity minus benchmark with benchmark opt-in, key-name secret masking.) diff --git a/specs/archive/003-collect-tool/tasks.md b/specs/archive/003-collect-tool/tasks.md new file mode 100644 index 0000000..7513184 --- /dev/null +++ b/specs/archive/003-collect-tool/tasks.md @@ -0,0 +1,162 @@ +# Tasks: Infrahub Collect (Troubleshooting Bundle Tool) + +**Input**: Design documents from `/specs/003-collect-tool/` + +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/ (cli.md, bundle-layout.md, manifest.schema.json), quickstart.md + +**Tests**: Included — constitution Principle IV mandates table-driven unit tests for pure logic and e2e coverage for container-touching flows. + +**Organization**: Tasks are grouped by user story. US1 (Kubernetes, P1) is the MVP; the environment-agnostic collector framework lands in Foundational, shared parity collectors land in US1 (earliest story that needs them) and run unchanged on Docker for US2. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies on incomplete tasks) +- **[Story]**: US1 / US2 / US3 (user story phases only) + +## Path Conventions + +Single Go project: binaries in `src/cmd//main.go`, all logic in `src/internal/app/`, e2e in `tests/e2e/`. Design references: `specs/003-collect-tool/`. + +--- + +## Phase 1: Setup (binary skeleton and build surface) + +**Purpose**: A buildable third binary with the full CLI surface wired (create initially a stub), so every later phase is testable via `make build`. + +- [X] T001 Create `src/cmd/infrahub-collect/main.go` mirroring `src/cmd/infrahub-taskmanager/main.go`: `app.SetVersion`, `app.NewInfrahubOps`, `app.ConfigureRootCommand`, `app.AttachEnvironmentCommands`, `version` command, and a `create` command (stub RunE) registering flags per `specs/003-collect-tool/contracts/cli.md` — persistent `--output-dir` (default `./infrahub_bundles`) and create-local `--log-lines` (default 100000), `--include-backup`, `--include-queries`, `--benchmark`, each viper-bound for `INFRAHUB_*` env equivalents +- [X] T002 [P] Add `infrahub-collect` to the `BINARIES` variable in `Makefile` (wires `build`, `build-all`, `install`) +- [X] T003 [P] Add an `infrahub-collect` `buildGoModule` package and symlinkJoin entry in `flake.nix` (no `vendorHash` change — `go.mod` untouched) +- [X] T004 Verify `make build` produces `bin/infrahub-collect` and that `--help`, `create --help`, `version`, `environment detect|list` match `specs/003-collect-tool/contracts/cli.md` + +**Checkpoint**: `bin/infrahub-collect` builds; CLI surface matches the published contract. + +--- + +## Phase 2: Foundational (collector framework — blocks all user stories) + +**Purpose**: Environment-agnostic core: masking, manifest, timeout-bounded execution, orchestrator, and the backend seam collectors consume. + +- [X] T005 [P] Implement `src/internal/app/masking.go`: pure key-name masking per research R5 — case-insensitive substring match on `password|secret|token|key` replaces values with `********`; helpers for `KEY=VALUE` env dumps and key/value config dumps (Redis `CONFIG GET`, `rabbitmqctl environment`/`status`) +- [X] T006 [P] Table-driven tests in `src/internal/app/masking_test.go` (match/no-match matrix, case-insensitivity, env-line and config-pair formats, empty values) +- [X] T007 [P] Implement `src/internal/app/collect_manifest.go`: `manifestVersion = 2026070200`, `BundleManifest` + `CollectorResult` + status constants per `specs/003-collect-tool/data-model.md`, constructor, and writer producing `bundle_information.json` via `json.MarshalIndent` (mirror `backup_metadata.go` conventions) +- [X] T008 [P] Table-driven tests in `src/internal/app/collect_manifest_test.go`: JSON field names/values validate against `specs/003-collect-tool/contracts/manifest.schema.json` semantics (collect_id pattern, status enum, reason required on failed/skipped) +- [X] T009 Define the collect backend seam in `src/internal/app/collect.go`: `Replica` struct (Service, Pod, Container, Restarted per data-model.md) and a narrow `collectBackend` interface (embeds `EnvironmentBackend`; adds `ServiceReplicas(service)`, `ReplicaLogs(replica, tailLines, previous)`, `Metrics()`) to be satisfied by both backends and a test fake +- [X] T010 Add context/timeout-bounded execution to `src/internal/app/command_executor.go` (`exec.CommandContext` variants of `runCommand`/`runCommandPipe`; research R2: 60s default for exec dumps, 5 min for log/copy transfers; timeout error distinguishable so the manifest reason reads `timed out after `) +- [X] T011 Implement the orchestrator `CollectBundle(opts CollectOptions)` in `src/internal/app/collect.go`: staging dir via `os.MkdirTemp(outputDir, ...)`, ordered collector loop with skip preconditions and non-fatal failures (WARN + manifest entry, FR-009), per-collector timeouts, per-collector INFO progress lines, manifest finalized last, archive via existing `createTarball(archivePath, workDir, "bundle/")`, staging cleanup on success/failure/SIGINT/SIGTERM, final log with archive path + size, hard error (exit 1 path) only for no-environment/unwritable-output/archive failure per contracts/cli.md +- [X] T012 Unit tests in `src/internal/app/collect_test.go` with a fake `collectBackend`: outcomes recorded for success/failed/skipped/timed-out collectors, run continues past failures, manifest accounts for every planned collector (SC-005), staging dir removed +- [X] T013 Wire the `create` command RunE in `src/cmd/infrahub-collect/main.go` to build `CollectOptions` from viper and call `iops.CollectBundle(opts)` + +**Checkpoint**: Framework complete — collectors can be added independently; user story phases can begin. + +--- + +## Phase 3: User Story 1 — Collect from a Kubernetes deployment (Priority: P1) 🎯 MVP + +**Goal**: One command against a Helm-chart deployment yields a complete bundle: all-replica logs (with previous-container logs), parity diagnostics, server info, metrics, manifest. + +**Independent Test**: Deploy Infrahub via the Helm chart into a test cluster, run `infrahub-collect create` with only a kubeconfig, verify per-service logs, diagnostics, server info, and manifest in the archive (spec US1 acceptance scenarios 1–4). + +### Implementation for User Story 1 + +- [X] T014 [US1] Implement `ServiceReplicas` on `KubernetesBackend` in `src/internal/app/environment_kubernetes.go`: existing `GetAllPods` + `kubectl get pod -o jsonpath` over `status.containerStatuses` → one `Replica` per pod **container** with per-container `restartCount` (research R3 / critique E2) +- [X] T015 [US1] Implement `ReplicaLogs` on `KubernetesBackend`: `kubectl logs -n -c --tail= [--previous]`, streamed via the timeout-bounded pipe primitive (T010) +- [X] T016 [P] [US1] Implement `Metrics` on `KubernetesBackend`: `kubectl top pods -n `; missing metrics-server surfaces as a normal error (manifest `failed`, research R4) +- [X] T017 [US1] Implement the service-log collector in `src/internal/app/collect_logs.go` (environment-agnostic via the T009 seam): iterate the canonical service list, per-replica files named per data-model.md (`.log` / `.log` / `_.log`), `.previous.log` when `Restarted`, per-service manifest entries `logs/`, absent optional services → `skipped` +- [X] T018 [P] [US1] Implement the database-log collector in `src/internal/app/collect_diagnostics.go`: `CopyFrom` Neo4j `neo4j.log` + `debug.log` into `bundle/database/`; with `--include-queries` copy the full log directory including query logs (FR-004) +- [X] T019 [P] [US1] Implement the message-queue collector in `src/internal/app/collect_diagnostics.go`: `rabbitmqctl list_queues / list_exchanges / list_bindings / list_connections / list_channels / status / environment` into `bundle/message-queue/`, masking `environment` and `status` output (research R5) +- [X] T020 [P] [US1] Implement the cache collector in `src/internal/app/collect_diagnostics.go`: `redis-cli info / client list / config get '*' (masked) / slowlog get / dbsize` into `bundle/cache/` +- [X] T021 [P] [US1] Implement the task-worker collector in `src/internal/app/collect_diagnostics.go`: Prefect worker CLI output per replica into `bundle/task-worker//` (uses T009 `ServiceReplicas`) +- [X] T022 [P] [US1] Implement the task-manager collector in `src/internal/app/collect_diagnostics.go`: work pools, work queues, recent flow runs, events, automations via Prefect CLI (embedded script via `script_executor.go` + `go:embed` where CLI output is insufficient) into `bundle/task-manager/` +- [X] T023 [P] [US1] Implement the server-info collector in `src/internal/app/collect_diagnostics.go`: existing `getInfrahubVersion`, `pip list`, masked `env` dump, API info/config/schema fetched inside `infrahub-server` against `infrahubInternalAddress` (research R8) into `bundle/server/` +- [X] T024 [US1] Implement the metrics collector in `src/internal/app/collect_metrics.go` using the backend `Metrics` primitive, output into `bundle/metrics/` +- [X] T025 [US1] Register the full ordered run plan in `CollectBundle` (`src/internal/app/collect.go`): logs per service → database → message-queue → cache → task-worker → task-manager → server → metrics (+ opt-in extras), populating `infrahub_version`, `environment`, `log_lines` in the manifest +- [X] T026 [US1] Unit tests in `src/internal/app/collect_logs_test.go`: log filename derivation (docker/k8s single/multi-container, previous), kubectl/docker argument construction, replica parsing from jsonpath output +- [X] T027 [US1] Add `collect_binary` session fixture in `tests/e2e/conftest.py` (mirrors `backup_binary`: `make build` → `bin/infrahub-collect`) and `run_collect` helper in `tests/helpers/utils.py` +- [X] T028 [US1] E2E test `tests/e2e/test_k8s_collect.py` (`-m k8s`): full collect against the kind+Helm stack — archive integrity, manifest validates against `specs/003-collect-tool/contracts/manifest.schema.json`, one log file per replica (scale task-worker ≥ 2), `.previous.log` for an induced restart, layout per `specs/003-collect-tool/contracts/bundle-layout.md`, zero pod restarts/scale events caused by the run (SC-003) + +**Checkpoint**: US1 fully functional — Kubernetes bundle collection works end-to-end (MVP). + +--- + +## Phase 4: User Story 2 — Collect from Docker Compose without the repo (Priority: P2) + +**Goal**: The same binary and command produce a parity bundle on Docker Compose; the shared collectors from US1 run unchanged through the Docker backend. + +**Independent Test**: Start an Infrahub compose project, run `infrahub-collect create` with only the binary and Docker installed, compare bundle contents against the parity list (spec US2 acceptance scenarios). + +### Implementation for User Story 2 + +- [X] T029 [US2] Implement `ServiceReplicas` on `DockerBackend` in `src/internal/app/environment_docker.go`: `docker compose -p ps ` → one `Replica` per container using the container **name** (critique P2) +- [X] T030 [US2] Implement `ReplicaLogs` on `DockerBackend`: `docker logs --tail ` via the timeout-bounded pipe primitive; `previous` never requested on Docker +- [X] T031 [P] [US2] Implement `Metrics` on `DockerBackend`: `docker stats --no-stream` over the project's containers +- [X] T032 [US2] E2E test `tests/e2e/test_docker_collect.py` (`-m docker`): full bundle with parity content and schema-validated manifest; project selection with/without `--project` (US2 scenario 2); degraded case — stop `cache`, expect exit 0 + `{"name": "cache-status", "status": "failed"}` (FR-009/SC-005); masked env output contains no plaintext secrets (FR-008); `--log-lines=500` and `INFRAHUB_LOG_LINES` precedence recorded in manifest (FR-011) + +**Checkpoint**: US1 and US2 both work; bundle layout identical across environments (SC-004). + +--- + +## Phase 5: User Story 3 — Include a backup for issue reproduction (Priority: P3) + +**Goal**: `--include-backup` produces a standard backup alongside the bundle in one run, recorded in the manifest; backup failure never loses the bundle. + +**Independent Test**: Run `infrahub-collect create --include-backup` against a test deployment; verify the backup artifact exists next to the bundle and is referenced in the manifest (spec US3 acceptance scenarios). + +### Implementation for User Story 3 + +- [X] T033 [US3] Implement the include-backup collector in `src/internal/app/collect_extras.go`: invoke existing `CreateBackup` unmodified (non-interactive defaults: no S3 upload, no redaction, no encryption — research R10), record the backup's identity/path in the manifest; backup failure → manifest `failed` + bundle still produced (US3 scenario 2) +- [X] T034 [US3] Unit test in `src/internal/app/collect_extras_test.go`: include-backup outcome recording for success and failure paths (fake backup runner seam), skipped when flag unset +- [X] T035 [US3] Extend `tests/e2e/test_docker_collect.py` with an `--include-backup` case: backup artifact created next to the bundle, manifest references it, collect exit 0 + +**Checkpoint**: All three user stories independently functional. + +--- + +## Phase 6: Opt-in benchmark (FR-013) + +**Purpose**: Cross-environment opt-in benchmark collector — not tied to a user story; required by FR-013 and the published `--benchmark` flag. + +- [X] T036 Implement the benchmark collector in `src/internal/app/collect_extras.go`: port the benchmark image reference from the Python `invoke bundle collect` implementation (critique Q1) with `INFRAHUB_BENCHMARK_IMAGE` override; Docker: `docker run` attached to the project network; Kubernetes: one-off `kubectl run` pod in the namespace; capture output to `bundle/benchmark/`; always delete the transient container/pod; image pull/run failure → manifest `skipped` + warning (research R11); default off → `skipped`/`not requested`; add an `INFRAHUB_BENCHMARK_IMAGE` row to `docs/docs/reference/configuration.mdx` (Vale/rumdl must pass) + +--- + +## Phase 7: Polish & Cross-Cutting Concerns + +- [X] T037 [P] Update `AGENTS.md` project overview and architecture sections from two to three binaries, adding `infrahub-collect` command summary (constitution v1.1.0 Sync Impact Report mandates this alongside implementation) +- [X] T038 [P] Wire collect e2e tests into the existing docker/k8s e2e jobs in `.github/workflows/ci.yml` (same markers/fixtures; critique E4) +- [X] T039 [P] Fix `docs/docs/guides/install-collect.mdx` prerequisite "Go 1.21 or later" → Go 1.25 (matches `go.mod`; Vale/rumdl must pass) +- [X] T040 Run `specs/003-collect-tool/quickstart.md` validation scenarios 1–4 and 6 locally (build, CLI smoke, Docker collection, degraded case, flag/env precedence) and full gates: `make fmt && make vet && make lint && make test` + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: no dependencies; T002/T003 parallel after T001 exists (T004 needs all three) +- **Foundational (Phase 2)**: depends on Setup; **blocks all user stories**. T005–T008 fully parallel; T009+T010 before T011; T011 before T012/T013 +- **US1 (Phase 3)**: depends on Foundational. T014 before T015; T017 needs T014/T015; T018–T023 parallel (same file `collect_diagnostics.go` for T018–T023 — parallel authoring only if split carefully; safe order is sequential within that file); T025 needs T017–T024; T027 before T028 +- **US2 (Phase 4)**: depends on Foundational + T017/T024/T025 (shared collectors from US1). T029 before T030; T032 needs T027 +- **US3 (Phase 5)**: depends on Foundational + T025 (run-plan registration); independent of US2 +- **Benchmark (Phase 6)**: depends on Foundational + T025 +- **Polish (Phase 7)**: T037–T039 anytime after their subjects stabilize; T040 last + +### User Story Dependencies + +- **US1 (P1)**: only Foundational — the MVP +- **US2 (P2)**: Foundational + the environment-agnostic collectors delivered in US1 (T017–T025); its own backend primitives are independent +- **US3 (P3)**: Foundational + run-plan registration (T025); no dependency on US2 + +### Parallel Opportunities + +- Phase 2: T005+T006, T007+T008 pairs in parallel (different files) +- Phase 3: T016 parallel with T014/T015; T018–T023 are distinct collectors (parallelize by pre-splitting `collect_diagnostics.go` sections or serialize) +- Phase 4: T031 parallel with T029/T030 +- Phase 7: T037, T038, T039 all parallel + +## Implementation Strategy + +**MVP first**: Phases 1–3 deliver US1 (Kubernetes) — the zero-alternative gap and the feature's core value. Stop, validate with quickstart scenario 5 / e2e `-m k8s -k collect`, then increment. + +**Incremental delivery**: Phase 4 turns on Docker parity (US2) with only three backend primitives + one e2e file. Phase 5 (US3) and Phase 6 (benchmark) are small, independent additions. Phase 7 closes the constitution follow-up (AGENTS.md), CI wiring, and full-gate validation. + +**Notes**: commit after each task or logical group; `make fmt && make vet && make lint && make test` must stay green at every checkpoint (Principle IV); no `go.mod` changes are expected — if one occurs, run `scripts/update-vendor-hash.sh` (constitution). diff --git a/src/cmd/infrahub-collect/main.go b/src/cmd/infrahub-collect/main.go new file mode 100644 index 0000000..037832a --- /dev/null +++ b/src/cmd/infrahub-collect/main.go @@ -0,0 +1,90 @@ +package main + +import ( + "fmt" + "os" + + app "infrahub-ops/src/internal/app" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// version is set via ldflags at build time +var version string + +func main() { + app.SetVersion(version) + iops := app.NewInfrahubOps() + rootCmd := &cobra.Command{ + Use: "infrahub-collect", + Short: "Collect Infrahub troubleshooting bundles", + Long: "Collect troubleshooting bundles (service logs, diagnostics, metrics) from Infrahub deployments running on Docker Compose or Kubernetes.", + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + + app.ConfigureRootCommand(rootCmd, iops) + app.AttachEnvironmentCommands(rootCmd, iops) + + // Collect-specific persistent flag (INFRAHUB_OUTPUT_DIR) + rootCmd.PersistentFlags().String("output-dir", "./infrahub_bundles", "Directory for bundle files") + viper.BindPFlag("output-dir", rootCmd.PersistentFlags().Lookup("output-dir")) + + var logLines int + var includeBackup bool + var includeQueries bool + var benchmark bool + + createCmd := &cobra.Command{ + Use: "create", + Short: "Collect a troubleshooting bundle from the Infrahub instance", + Long: "Collect a troubleshooting bundle from the Infrahub instance. Collection is read-only: no container or pod is stopped, restarted, or scaled. Individual collector failures are recorded in the bundle manifest and do not abort the run.", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + lines := viper.GetInt("log-lines") + if lines <= 0 { + return fmt.Errorf("--log-lines must be a positive integer, got %d", lines) + } + + opts := app.CollectOptions{ + OutputDir: viper.GetString("output-dir"), + LogLines: lines, + IncludeBackup: viper.GetBool("include-backup"), + IncludeQueries: viper.GetBool("include-queries"), + Benchmark: viper.GetBool("benchmark"), + } + + return iops.CollectBundle(opts) + }, + } + createCmd.Flags().IntVar(&logLines, "log-lines", 100000, "Maximum log lines collected per container") + createCmd.Flags().BoolVar(&includeBackup, "include-backup", false, "Also create a backup using the standard backup behavior") + createCmd.Flags().BoolVar(&includeQueries, "include-queries", false, "Include database query logs (may contain customer data)") + createCmd.Flags().BoolVar(&benchmark, "benchmark", false, "Run the OpsMill benchmark and include its results (requires image download; skipped with a warning if unavailable)") + + // Bind create flags to Viper for environment variable support (INFRAHUB_) + viper.BindPFlag("log-lines", createCmd.Flags().Lookup("log-lines")) + viper.BindPFlag("include-backup", createCmd.Flags().Lookup("include-backup")) + viper.BindPFlag("include-queries", createCmd.Flags().Lookup("include-queries")) + viper.BindPFlag("benchmark", createCmd.Flags().Lookup("benchmark")) + + rootCmd.AddCommand(createCmd) + + versionCmd := &cobra.Command{ + Use: "version", + Short: "Print Infrahub Ops CLI build information", + Run: func(cmd *cobra.Command, args []string) { + logrus.Infof("Version: %s", app.BuildRevision()) + }, + } + + rootCmd.AddCommand(versionCmd) + + if err := rootCmd.Execute(); err != nil { + logrus.Errorf("Command failed: %v", err) + os.Exit(1) + } +} diff --git a/src/internal/app/collect.go b/src/internal/app/collect.go new file mode 100644 index 0000000..aed31a1 --- /dev/null +++ b/src/internal/app/collect.go @@ -0,0 +1,315 @@ +package app + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/sirupsen/logrus" +) + +// Per-collector subprocess time bounds (research R2): exec/status dumps get +// 60s, log downloads and file copies get 5 minutes. The opt-in benchmark gets +// its own generous bound: it downloads an image (--pull always) and then runs +// disk and CPU measurements, which can exceed the transfer bound on slow +// disks or links. +const ( + collectExecTimeout = 60 * time.Second + collectTransferTimeout = 5 * time.Minute + collectBenchmarkTimeout = 10 * time.Minute +) + +// CollectOptions aggregates the create-command inputs resolved from +// flags/environment variables by the entry point. +type CollectOptions struct { + OutputDir string + LogLines int + IncludeBackup bool + IncludeQueries bool + Benchmark bool +} + +// Replica is a read-only descriptor of one running container of a service. +// On Docker, Container carries the container name and Pod is empty; on +// Kubernetes there is one Replica per pod container, and Restarted reports +// whether that container has restartCount > 0 (previous logs available). +type Replica struct { + Service string + Pod string + Container string + Restarted bool +} + +// collectBackend is the narrow read-only surface collectors consume on top of +// the shared EnvironmentBackend. Both DockerBackend and KubernetesBackend +// satisfy it once their replica/log/metrics primitives are implemented; unit +// tests provide a fake. +type collectBackend interface { + EnvironmentBackend + // ServiceReplicas enumerates the running replicas of a service. + ServiceReplicas(service string) ([]Replica, error) + // ReplicaLogs streams up to tailLines of a replica's logs; previous + // requests the prior container's logs (Kubernetes only). The caller must + // drain the reader and then call the wait function. + ReplicaLogs(replica Replica, tailLines int, previous bool) (io.ReadCloser, func() error, error) + // Metrics returns one-shot resource metrics for the deployment + // (docker stats / kubectl top). + Metrics() (string, error) +} + +// collectContext carries the per-run state handed to each collector. +type collectContext struct { + ctx context.Context + iops *InfrahubOps + backend EnvironmentBackend + opts CollectOptions + bundleDir string + // manifest lets collectors contribute run-level fields (e.g. the + // server-info collector fills infrahub_version best-effort). Outcome + // entries stay the orchestrator's job. + manifest *BundleManifest + // artifact is a filesystem reference handed off by the running collector + // (e.g. the --include-backup archive path); the orchestrator moves it + // into that collector's manifest entry after a successful run. + artifact string +} + +// setArtifact hands the orchestrator a filesystem artifact reference to +// record on the current collector's manifest entry. +func (cc *collectContext) setArtifact(path string) { + cc.artifact = path +} + +// takeArtifact returns and clears the pending artifact reference so it never +// leaks into a later collector's entry. +func (cc *collectContext) takeArtifact() string { + artifact := cc.artifact + cc.artifact = "" + return artifact +} + +// collect resolves the active environment backend to the collect-side +// interface. Collectors that need replica/log/metrics primitives call this +// and surface a clear per-collector failure when the backend does not +// implement them. +func (cc *collectContext) collect() (collectBackend, error) { + backend, ok := cc.backend.(collectBackend) + if !ok { + return nil, fmt.Errorf("%s environment does not support bundle collection primitives", cc.backend.Name()) + } + return backend, nil +} + +// collector is the internal unit of collection: a named step that writes +// files under the staging bundle directory. skip is an optional precondition +// returning (true, reason) when the collector does not apply to this run. +type collector struct { + name string + skip func(cc *collectContext) (bool, string) + run func(cc *collectContext) error +} + +// collectSkipError marks a collector run outcome that must be recorded as +// skipped — with a warning reason — instead of failed: the opt-in benchmark +// degrades this way when its image cannot be pulled or run, so air-gapped +// runs stay clean (research R11, spec air-gapped edge case). +type collectSkipError struct{ reason string } + +func (e *collectSkipError) Error() string { return e.reason } + +// CollectBundle gathers a troubleshooting bundle from the detected +// environment into /support_bundle_.tar.gz. Individual +// collector failures are recorded in the bundle manifest and never abort the +// run (FR-009); only a missing environment, an unwritable output directory, +// or an archiving failure returns an error. +func (iops *InfrahubOps) CollectBundle(opts CollectOptions) error { + backend, err := iops.ensureBackend() + if err != nil { + return fmt.Errorf("bundle collection requires a usable Infrahub environment: %w", err) + } + + return iops.runCollectPlan(backend, opts, iops.collectPlan(opts)) +} + +// collectPlan builds the ordered collector run plan for this run: logs per +// canonical service, then the parity diagnostics (database → message-queue → +// cache → task-worker → task-manager → server), then metrics (research R9). +// The manifest's environment and log_lines are set at construction; the +// server-info collector fills infrahub_version best-effort. +func (iops *InfrahubOps) collectPlan(opts CollectOptions) []collector { + plan := serviceLogCollectors() + plan = append(plan, + databaseLogsCollector(), + messageQueueCollector(), + cacheCollector(), + taskWorkerCollector(), + taskManagerCollector(), + serverInfoCollector(), + metricsCollector(), + ) + + // Opt-in extras run last so the always-on diagnostics are already staged + // when they start. The benchmark generates load, so it runs only after + // every read-only collector has captured the undisturbed state; the + // include-backup collector inherits the standard backup behavior — it may + // stop/restart application containers — so it must stay last of all. + plan = append(plan, + benchmarkCollector(runEnvironmentBenchmark), + includeBackupCollector(runStandardBackup), + ) + return plan +} + +// safeRun executes a collector's run function, converting a panic into a +// recorded failure so one misbehaving collector never aborts the whole run +// (FR-009, FIX-3). +func safeRun(c collector, cc *collectContext) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + return c.run(cc) +} + +// safeSkip evaluates a collector's optional skip precondition, converting a +// panic into an error so the orchestrator records the collector as failed +// rather than crashing the whole run (FIX-3). +func safeSkip(c collector, cc *collectContext) (skip bool, reason string, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + if c.skip == nil { + return false, "", nil + } + skip, reason = c.skip(cc) + return skip, reason, nil +} + +// runCollectPlan stages the bundle, runs every collector in order, finalizes +// the manifest, and packages the archive. The staging directory is removed on +// success, failure, and interrupt (SIGINT/SIGTERM). +func (iops *InfrahubOps) runCollectPlan(backend EnvironmentBackend, opts CollectOptions, plan []collector) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := os.MkdirAll(opts.OutputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory %s: %w", opts.OutputDir, err) + } + + workDir, err := os.MkdirTemp(opts.OutputDir, "infrahub_collect_*") + if err != nil { + return fmt.Errorf("failed to create staging directory in %s: %w", opts.OutputDir, err) + } + defer func() { + if removeErr := os.RemoveAll(workDir); removeErr != nil { + logrus.Warnf("Failed to remove staging directory %s: %v", workDir, removeErr) + } + }() + + bundleDir := filepath.Join(workDir, "bundle") + if err := os.MkdirAll(bundleDir, 0755); err != nil { + return fmt.Errorf("failed to create staging bundle directory %s: %w", bundleDir, err) + } + + collectID := generateCollectID() + archivePath := filepath.Join(opts.OutputDir, fmt.Sprintf("support_bundle_%s.tar.gz", collectID)) + manifest := newBundleManifest(collectID, backend.Name(), opts.LogLines) + + logrus.WithFields(logrus.Fields{ + "collect_id": collectID, + "output_dir": opts.OutputDir, + "environment": backend.Name(), + }).Info("Collecting troubleshooting bundle") + + cc := &collectContext{ + ctx: ctx, + iops: iops, + backend: backend, + opts: opts, + bundleDir: bundleDir, + manifest: manifest, + } + + for _, c := range plan { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("bundle collection interrupted: %w", ctxErr) + } + + if skip, reason, skipErr := safeSkip(c, cc); skipErr != nil { + // A panicking skip precondition must not abort the run (FR-009): + // record it as a failure and continue. + logrus.Warnf("Collector %s failed: %v", c.name, skipErr) + manifest.recordFailed(c.name, skipErr.Error()) + continue + } else if skip { + logrus.Infof("Skipping %s: %s", c.name, reason) + manifest.recordSkipped(c.name, reason) + continue + } + + logrus.Infof("Collecting %s", c.name) + if err := safeRun(c, cc); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("bundle collection interrupted: %w", ctxErr) + } + + var skip *collectSkipError + if errors.As(err, &skip) { + logrus.Warnf("Collector %s skipped: %s", c.name, skip.reason) + cc.takeArtifact() // drop a skipped collector's artifact reference + manifest.recordSkipped(c.name, skip.reason) + continue + } + + reason := err.Error() + var timeout *timeoutError + if errors.As(err, &timeout) { + reason = timeout.Error() + } + logrus.Warnf("Collector %s failed: %v", c.name, err) + cc.takeArtifact() // drop a failed collector's artifact reference + manifest.recordFailed(c.name, reason) + continue + } + if artifact := cc.takeArtifact(); artifact != "" { + manifest.recordSuccessArtifact(c.name, artifact) + } else { + manifest.recordSuccess(c.name) + } + } + + // The manifest is finalized last so it reflects every collector outcome. + if err := manifest.write(bundleDir); err != nil { + return err + } + + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("bundle collection interrupted: %w", ctxErr) + } + + logrus.Info("Creating bundle archive...") + if err := createTarball(archivePath, workDir, "bundle/"); err != nil { + if removeErr := os.Remove(archivePath); removeErr != nil && !os.IsNotExist(removeErr) { + logrus.Warnf("Failed to remove partial archive %s: %v", archivePath, removeErr) + } + return fmt.Errorf("failed to create bundle archive %s: %w", archivePath, err) + } + + fields := logrus.Fields{"path": archivePath} + if stat, statErr := os.Stat(archivePath); statErr == nil { + fields["size_bytes"] = stat.Size() + fields["size_human"] = formatBytes(stat.Size()) + } + logrus.WithFields(fields).Info("Bundle created successfully") + + return nil +} diff --git a/src/internal/app/collect_diagnostics.go b/src/internal/app/collect_diagnostics.go new file mode 100644 index 0000000..d1cc49c --- /dev/null +++ b/src/internal/app/collect_diagnostics.go @@ -0,0 +1,616 @@ +package app + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// Bundle directory names for the parity diagnostics collectors +// (specs/003-collect-tool/contracts/bundle-layout.md). +const ( + databaseBundleDir = "database" + messageQueueBundleDir = "message-queue" + cacheBundleDir = "cache" + taskWorkerBundleDir = "task-worker" + taskManagerBundleDir = "task-manager" + serverBundleDir = "server" +) + +// neo4jLogPath is where the official Neo4j image writes its server logs. +const neo4jLogPath = "/logs" + +// prefectServerAPIFallback is used when the task-manager container does not +// define PREFECT_API_URL: the Prefect server serves its API on port 4200 by +// default, and the command runs inside that container. +const prefectServerAPIFallback = "http://localhost:4200/api" + +// infrahubServerAPIFallback is used when INFRAHUB_INTERNAL_ADDRESS is not +// discoverable: the fetch runs inside the infrahub-server container, where +// the API listens on its default port. +const infrahubServerAPIFallback = "http://localhost:8000" + +// prefectEventsScript is the embedded script dumping recent Prefect events +// (the Prefect CLI has no non-streaming events command; research R9). +const ( + prefectEventsScript = "collect_prefect_events.py" + prefectEventsScriptTarget = "/tmp/infrahubops_collect_prefect_events.py" +) + +// contextExecer is an optional backend capability: a timeout-bounded variant +// of Exec (research R2: 60s per status dump). Both concrete backends +// implement it; collectors fall back to plain Exec on backends (or test +// fakes) that do not. +type contextExecer interface { + ExecContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) (string, error) +} + +// replicaExecer is an optional backend capability: execute a command in one +// specific replica instead of the backend's default pod/container for the +// service. KubernetesBackend implements it; on backends without it the +// collectors fall back to service-level Exec (single-replica behavior). +type replicaExecer interface { + ExecReplica(ctx context.Context, timeout time.Duration, replica Replica, command []string) (string, error) +} + +// contextCopier is an optional backend capability: a timeout-bounded variant of +// CopyFrom (research R2, FIX-1). Both concrete backends implement it; the +// collect path must use it so a wedged container/daemon cannot hang the run on +// an unbounded file copy. There is no fallback to the unbounded CopyFrom. +type contextCopier interface { + CopyFromContext(ctx context.Context, timeout time.Duration, service, src, dest string) error +} + +// copyFrom copies a file out of a service container bounded by the collect +// transfer timeout. A backend without the bounded-copy capability is a +// per-collector failure (recorded in the manifest) rather than an unbounded +// hang. +func (cc *collectContext) copyFrom(service, src, dest string) error { + copier, ok := cc.backend.(contextCopier) + if !ok { + return fmt.Errorf("%s environment does not support bounded file copy for bundle collection", cc.backend.Name()) + } + return copier.CopyFromContext(cc.ctx, collectTransferTimeout, service, src, dest) +} + +// captureTimeout returns the first *timeoutError seen: the already-captured one +// if present, otherwise the one unwrapped from err (or nil). Aggregating +// collectors use it to preserve a command timeout through string aggregation so +// the orchestrator can normalize the manifest reason to the bare +// "timed out after " contract string (research R2, FIX-4). +func captureTimeout(existing *timeoutError, err error) *timeoutError { + if existing != nil { + return existing + } + var t *timeoutError + if errors.As(err, &t) { + return t + } + return nil +} + +// partialError builds an aggregated partial-failure error from per-item +// reasons. When any sub-failure was a command timeout, the *timeoutError is +// wrapped with %w so the orchestrator's errors.As normalization reports the +// bare "timed out after " reason (research R2, FIX-4). Non-timeout +// partial failures keep their per-item reasons verbatim. +func partialError(prefix string, failures []string, timeout *timeoutError) error { + if len(failures) == 0 { + return nil + } + joined := strings.Join(failures, "; ") + if timeout != nil { + return fmt.Errorf("%s: %s: %w", prefix, joined, timeout) + } + return fmt.Errorf("%s: %s", prefix, joined) +} + +// execDump runs a command inside a service container, bounded by the collect +// exec timeout when the backend supports it. +func (cc *collectContext) execDump(service string, command []string) (string, error) { + if execer, ok := cc.backend.(contextExecer); ok { + return execer.ExecContext(cc.ctx, collectExecTimeout, service, command, nil) + } + return cc.backend.Exec(service, command, nil) +} + +// execReplicaDump runs a command inside one specific replica, falling back to +// service-level exec when the backend cannot target replicas. +func (cc *collectContext) execReplicaDump(replica Replica, command []string) (string, error) { + if execer, ok := cc.backend.(replicaExecer); ok { + return execer.ExecReplica(cc.ctx, collectExecTimeout, replica, command) + } + return cc.execDump(replica.Service, command) +} + +// skipWhenServiceAbsent mirrors serviceLogCollector's precondition: a service +// with zero replicas is not deployed in this environment, so the collector is +// skipped. Enumeration errors (and backends without the collect primitives) +// leave the decision to run(), which surfaces real failures as failed. +func skipWhenServiceAbsent(service string) func(cc *collectContext) (bool, string) { + return func(cc *collectContext) (bool, string) { + cb, err := cc.collect() + if err != nil { + return false, "" + } + replicas, err := cb.ServiceReplicas(service) + if err != nil { + return false, "" + } + if len(replicas) == 0 { + return true, "service not deployed" + } + return false, "" + } +} + +// execDumpSpec describes one diagnostic dump: the command executed inside the +// service container and the bundle file its output is written to. mask, when +// set, is applied to the output before it is written (research R5). +type execDumpSpec struct { + filename string + command []string + mask func(string) string +} + +// writeDumpFile writes one dump to the bundle, masking it first when the spec +// requires it. +func writeDumpFile(path, content string, mask func(string) string) error { + if mask != nil { + content = mask(content) + } + if !strings.HasSuffix(content, "\n") { + content += "\n" + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} + +// execDumpsInto runs each dump inside service and writes the outputs into +// dir, returning one failure string per dump that could not be fully +// collected plus the first command timeout observed (FIX-4). A failing +// command's partial output (often the tool's own error text) is still written +// — masked — so the bundle shows what the service reported (FR-009). +func (cc *collectContext) execDumpsInto(service, dir string, dumps []execDumpSpec) ([]string, *timeoutError) { + failures := []string{} + var timeout *timeoutError + for _, dump := range dumps { + output, err := cc.execDump(service, dump.command) + if err != nil { + logrus.Warnf("Failed to run %q in %s: %v", strings.Join(dump.command, " "), service, err) + failures = append(failures, fmt.Sprintf("%s: %v", dump.filename, err)) + timeout = captureTimeout(timeout, err) + if strings.TrimSpace(output) == "" { + continue + } + } + if writeErr := writeDumpFile(filepath.Join(dir, dump.filename), output, dump.mask); writeErr != nil { + failures = append(failures, writeErr.Error()) + } + } + return failures, timeout +} + +// runExecDumps creates bundle// and collects the dumps into it, +// returning the aggregated failures and the first command timeout observed. +func (cc *collectContext) runExecDumps(service, dirName string, dumps []execDumpSpec) ([]string, *timeoutError) { + dir := filepath.Join(cc.bundleDir, dirName) + if err := os.MkdirAll(dir, 0755); err != nil { + return []string{fmt.Sprintf("failed to create %s directory: %v", dirName, err)}, nil + } + return cc.execDumpsInto(service, dir, dumps) +} + +// execDumpCollector builds a collector that runs a fixed set of exec dumps +// inside one service. Partial failures keep the successful dumps in the +// bundle and mark the collector failed with the aggregated reasons; a command +// timeout is wrapped so the manifest reason collapses to the bare timeout +// string (FIX-4). +func execDumpCollector(name, service, dirName string, dumps func() []execDumpSpec) collector { + return collector{ + name: name, + skip: skipWhenServiceAbsent(service), + run: func(cc *collectContext) error { + failures, timeout := cc.runExecDumps(service, dirName, dumps()) + return partialError(fmt.Sprintf("partial %s collection", dirName), failures, timeout) + }, + } +} + +// --- database (T018) --- + +// defaultDatabaseLogFiles are the Neo4j server logs always collected; query +// logs join them only with --include-queries (FR-004). +var defaultDatabaseLogFiles = []string{"neo4j.log", "debug.log"} + +// databaseLogsCollector copies the Neo4j server logs into bundle/database/. +func databaseLogsCollector() collector { + return collector{ + name: "database-logs", + skip: skipWhenServiceAbsent("database"), + run: collectDatabaseLogs, + } +} + +func collectDatabaseLogs(cc *collectContext) error { + dir := filepath.Join(cc.bundleDir, databaseBundleDir) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create %s directory: %w", databaseBundleDir, err) + } + + failures := []string{} + var timeout *timeoutError + files := defaultDatabaseLogFiles + if cc.opts.IncludeQueries { + // The full log directory includes rotated query logs + // (query.log, query.log.1, ...), so it is enumerated live. + names, err := cc.listDatabaseLogFiles() + switch { + case err != nil: + failures = append(failures, fmt.Sprintf("failed to list %s: %v", neo4jLogPath, err)) + timeout = captureTimeout(timeout, err) + case len(names) == 0: + // Enumeration returned nothing (empty ls output): fall back to the + // default server logs so the collector never records an empty + // database/ directory as success (FIX-7). + logrus.Warnf("Neo4j log directory %s enumerated empty; falling back to the default log set", neo4jLogPath) + default: + files = names + } + } + + for _, name := range files { + src := neo4jLogPath + "/" + name + if err := cc.copyFrom("database", src, filepath.Join(dir, name)); err != nil { + logrus.Warnf("Failed to copy %s from database: %v", src, err) + failures = append(failures, fmt.Sprintf("%s: %v", name, err)) + timeout = captureTimeout(timeout, err) + } + } + + return partialError("partial database log collection", failures, timeout) +} + +// listDatabaseLogFiles enumerates the files in the Neo4j log directory. +func (cc *collectContext) listDatabaseLogFiles() ([]string, error) { + output, err := cc.execDump("database", []string{"sh", "-c", "cd " + neo4jLogPath + " && ls -1"}) + if err != nil { + return nil, err + } + return nonEmptyLines(output), nil +} + +// --- message-queue (T019) --- + +// messageQueueDumps lists the RabbitMQ parity dumps (research R9), one file +// per dump; environment and status pass through Erlang-config masking +// (research R5). +func messageQueueDumps() []execDumpSpec { + return []execDumpSpec{ + {filename: "queues.txt", command: []string{"rabbitmqctl", "list_queues"}}, + {filename: "exchanges.txt", command: []string{"rabbitmqctl", "list_exchanges"}}, + {filename: "bindings.txt", command: []string{"rabbitmqctl", "list_bindings"}}, + {filename: "connections.txt", command: []string{"rabbitmqctl", "list_connections"}}, + {filename: "channels.txt", command: []string{"rabbitmqctl", "list_channels"}}, + {filename: "status.txt", command: []string{"rabbitmqctl", "status"}, mask: maskErlangConfig}, + {filename: "environment.txt", command: []string{"rabbitmqctl", "environment"}, mask: maskErlangConfig}, + } +} + +func messageQueueCollector() collector { + return execDumpCollector("message-queue-status", "message-queue", messageQueueBundleDir, messageQueueDumps) +} + +// --- cache (T020) --- + +// cacheDumps lists the Redis parity dumps (research R9); the configuration +// dump passes through key/value-pair masking (research R5). redis-cli runs +// unauthenticated, matching the default deployment; a password-protected +// Redis surfaces its NOAUTH error in the dump files. +func cacheDumps() []execDumpSpec { + return []execDumpSpec{ + {filename: "info.txt", command: []string{"redis-cli", "info"}}, + {filename: "clients.txt", command: []string{"redis-cli", "client", "list"}}, + {filename: "config.txt", command: []string{"redis-cli", "config", "get", "*"}, mask: maskConfigPairs}, + {filename: "slowlog.txt", command: []string{"redis-cli", "slowlog", "get"}}, + {filename: "dbsize.txt", command: []string{"redis-cli", "dbsize"}}, + } +} + +func cacheCollector() collector { + return execDumpCollector("cache-status", "cache", cacheBundleDir, cacheDumps) +} + +// --- task-worker (T021) --- + +// taskWorkerDumps lists the per-replica Prefect worker dumps. The Prefect +// worker exposes no dedicated status subcommand, so parity is the CLI +// version, the effective configuration (masked — it can carry API keys), and +// the work pools visible to the worker. +func taskWorkerDumps() []execDumpSpec { + return []execDumpSpec{ + {filename: "version.txt", command: []string{"prefect", "version"}}, + {filename: "config.txt", command: []string{"prefect", "config", "view"}, mask: maskEnvOutput}, + {filename: "work-pools.txt", command: []string{"prefect", "work-pool", "ls"}}, + } +} + +// taskWorkerCollector captures Prefect worker CLI output per replica into +// bundle/task-worker//. +func taskWorkerCollector() collector { + return collector{ + name: "task-worker-state", + skip: skipWhenServiceAbsent("task-worker"), + run: collectTaskWorkerState, + } +} + +func collectTaskWorkerState(cc *collectContext) error { + cb, err := cc.collect() + if err != nil { + return err + } + replicas, err := cb.ServiceReplicas("task-worker") + if err != nil { + return fmt.Errorf("failed to enumerate task-worker replicas: %w", err) + } + if len(replicas) == 0 { + return fmt.Errorf("no replicas found for service task-worker") + } + + counts := podContainerCounts(replicas) + failures := []string{} + var timeout *timeoutError + collected := 0 + for _, replica := range replicas { + replicaName := replicaBaseName(replica, counts[replica.Pod] > 1) + replicaDir := filepath.Join(cc.bundleDir, taskWorkerBundleDir, replicaName) + if err := os.MkdirAll(replicaDir, 0755); err != nil { + failures = append(failures, fmt.Sprintf("%s: %v", replicaName, err)) + continue + } + for _, dump := range taskWorkerDumps() { + output, err := cc.execReplicaDump(replica, dump.command) + if err != nil { + logrus.Warnf("Failed to run %q in %s: %v", strings.Join(dump.command, " "), replicaName, err) + failures = append(failures, fmt.Sprintf("%s/%s: %v", replicaName, dump.filename, err)) + timeout = captureTimeout(timeout, err) + // A failing CLI often prints the interesting error itself; + // keep it in the bundle when there is any output. + if strings.TrimSpace(output) == "" { + continue + } + } else { + collected++ + } + if writeErr := writeDumpFile(filepath.Join(replicaDir, dump.filename), output, dump.mask); writeErr != nil { + failures = append(failures, writeErr.Error()) + } + } + } + + // Per research R9 the collector is failed only when nothing at all could + // be collected; partial output is a success with warnings (the files show + // which commands failed). + if collected == 0 { + return partialError("task-worker state collection produced no output", failures, timeout) + } + if len(failures) > 0 { + logrus.Warnf("Partial task-worker state collection: %s", strings.Join(failures, "; ")) + } + return nil +} + +// --- task-manager (T022) --- + +// prefectServerCommand wraps a command so it runs with PREFECT_API_URL +// defaulting to the local Prefect server; a value already present in the +// container environment wins over the fallback. +func prefectServerCommand(command ...string) []string { + script := fmt.Sprintf(`export PREFECT_API_URL="${PREFECT_API_URL:-%s}"; exec "$@"`, prefectServerAPIFallback) + return append([]string{"sh", "-c", script, "sh"}, command...) +} + +// taskManagerDumps lists the Prefect server state dumps collected via the +// Prefect CLI inside the task-manager container (research R9). Recent events +// have no CLI equivalent and are collected separately via an embedded script. +func taskManagerDumps() []execDumpSpec { + return []execDumpSpec{ + {filename: "work-pools.txt", command: prefectServerCommand("prefect", "work-pool", "ls")}, + {filename: "work-queues.txt", command: prefectServerCommand("prefect", "work-queue", "ls")}, + {filename: "flow-runs.txt", command: prefectServerCommand("prefect", "flow-run", "ls", "--limit", "200")}, + {filename: "automations.txt", command: prefectServerCommand("prefect", "automation", "ls")}, + } +} + +// taskManagerCollector captures work pools, work queues, recent flow runs, +// automations, and recent events into bundle/task-manager/. +func taskManagerCollector() collector { + return collector{ + name: "task-manager-state", + skip: skipWhenServiceAbsent("task-manager"), + run: collectTaskManagerState, + } +} + +func collectTaskManagerState(cc *collectContext) error { + failures, timeout := cc.runExecDumps("task-manager", taskManagerBundleDir, taskManagerDumps()) + + dir := filepath.Join(cc.bundleDir, taskManagerBundleDir) + output, err := cc.runEmbeddedScriptDump("task-manager", prefectEventsScript, prefectEventsScriptTarget) + if err != nil { + logrus.Warnf("Failed to collect Prefect events: %v", err) + failures = append(failures, fmt.Sprintf("events.json: %v", err)) + timeout = captureTimeout(timeout, err) + // The script runs under CombinedOutput, so a failure merges a Python + // traceback into output. Writing that to events.json would masquerade + // as a valid events dump (FIX-8); preserve it in a sibling .err.txt for + // support instead. The failed reason is already recorded above. + if strings.TrimSpace(output) != "" { + if writeErr := writeDumpFile(filepath.Join(dir, "events.err.txt"), output, nil); writeErr != nil { + failures = append(failures, writeErr.Error()) + } + } + } else if writeErr := writeDumpFile(filepath.Join(dir, "events.json"), output, nil); writeErr != nil { + failures = append(failures, writeErr.Error()) + } + + return partialError("partial task-manager state collection", failures, timeout) +} + +// runEmbeddedScriptDump copies an embedded Python script into a service +// container, runs it under the collect exec timeout, and removes it again. +// Unlike executeScriptWithOpts it captures the output quietly instead of +// streaming it to the console — dump payloads belong in the bundle, not the +// progress log. +func (cc *collectContext) runEmbeddedScriptDump(service, scriptName, targetPath string) (string, error) { + scriptContent, err := readEmbeddedScript(scriptName) + if err != nil { + return "", fmt.Errorf("could not retrieve %s: %w", scriptName, err) + } + + tmpFile, err := os.CreateTemp("", "infrahubops_collect_*.py") + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + defer os.Remove(tmpFile.Name()) + if _, err := tmpFile.Write(scriptContent); err != nil { + tmpFile.Close() + return "", fmt.Errorf("failed to write script: %w", err) + } + tmpFile.Close() + + if err := cc.backend.CopyTo(service, tmpFile.Name(), targetPath); err != nil { + return "", fmt.Errorf("failed to copy %s to %s: %w", scriptName, service, err) + } + defer func() { + if _, err := cc.backend.Exec(service, []string{"rm", "-f", targetPath}, nil); err != nil { + logrus.Warnf("Failed to clean up script %s on %s: %v", targetPath, service, err) + } + }() + + return cc.execDump(service, []string{"python", "-u", targetPath}) +} + +// --- server (T023) --- + +// serverAPITarget maps one Infrahub API document to its bundle file. +type serverAPITarget struct { + filename string + path string + mask func(string) string +} + +// serverAPITargets lists the API documents captured into bundle/server/. The +// configuration dump is masked (research R5); info and schema carry no +// credentials. Paths that a given Infrahub version does not serve produce an +// observable HTTP error in the file and a failed reason in the manifest. +func serverAPITargets() []serverAPITarget { + return []serverAPITarget{ + {filename: "info.json", path: "/api/info"}, + {filename: "config.json", path: "/api/config", mask: maskJSON}, + {filename: "schema.json", path: "/api/schema"}, + } +} + +// serverAPIFetchCommand builds the Python command fetching one API document +// from inside the infrahub-server container (the image ships Python and +// httpx, not necessarily curl; research R8). The body is printed as-is; a +// non-2xx status is appended on stderr and reported via the exit code so the +// collector records the failure while the response stays observable. +func serverAPIFetchCommand(url string) []string { + script := strings.Join([]string{ + "import sys", + "import httpx", + "resp = httpx.get(sys.argv[1], timeout=30)", + "sys.stdout.write(resp.text)", + "if resp.status_code >= 400:", + " sys.stderr.write('\\nHTTP %d\\n' % resp.status_code)", + " sys.exit(1)", + }, "\n") + return []string{"python", "-c", script, url} +} + +// serverInfoCollector captures the Infrahub version, installed packages, +// masked environment, and API info/config/schema into bundle/server/. +func serverInfoCollector() collector { + return collector{ + name: "server-info", + skip: skipWhenServiceAbsent("infrahub-server"), + run: collectServerInfo, + } +} + +// collectInfrahubVersion detects the Infrahub version bounded by the collect +// exec timeout (FIX-5). It mirrors iops.getInfrahubVersion but goes through the +// bounded execDump so a wedged infrahub-server cannot hang the run; it does not +// touch the shared unbounded path the backup tool uses. +func (cc *collectContext) collectInfrahubVersion() string { + output, err := cc.execDump("infrahub-server", []string{"python", "-c", "import infrahub; print(infrahub.__version__)"}) + if err != nil { + logrus.Warnf("Could not detect Infrahub version: %v", err) + return "unknown" + } + return strings.TrimSpace(output) +} + +// collectInfrahubInternalAddress fetches INFRAHUB_INTERNAL_ADDRESS from the +// task-worker container bounded by the collect exec timeout (FIX-5). Empty +// string when unset or unreachable. Unlike iops.getInfrahubInternalAddress it +// never runs an unbounded exec and does not populate the shared cache. +func (cc *collectContext) collectInfrahubInternalAddress() string { + output, err := cc.execDump("task-worker", []string{"printenv", "INFRAHUB_INTERNAL_ADDRESS"}) + if err != nil { + logrus.Debugf("INFRAHUB_INTERNAL_ADDRESS not set in task-worker container: %v", err) + return "" + } + return strings.TrimSpace(output) +} + +func collectServerInfo(cc *collectContext) error { + dir := filepath.Join(cc.bundleDir, serverBundleDir) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create %s directory: %w", serverBundleDir, err) + } + + failures := []string{} + + version := cc.collectInfrahubVersion() + if version != "" && version != "unknown" && cc.manifest != nil { + cc.manifest.InfrahubVersion = version + } + if err := writeDumpFile(filepath.Join(dir, "version.txt"), version, nil); err != nil { + failures = append(failures, err.Error()) + } + + dumps := []execDumpSpec{ + {filename: "packages.txt", command: []string{"pip", "list"}}, + {filename: "environment.txt", command: []string{"env"}, mask: maskEnvOutput}, + } + + baseURL := cc.collectInfrahubInternalAddress() + if baseURL == "" { + baseURL = infrahubServerAPIFallback + } + baseURL = strings.TrimRight(baseURL, "/") + for _, target := range serverAPITargets() { + dumps = append(dumps, execDumpSpec{ + filename: target.filename, + command: serverAPIFetchCommand(baseURL + target.path), + mask: target.mask, + }) + } + + dumpFailures, timeout := cc.execDumpsInto("infrahub-server", dir, dumps) + failures = append(failures, dumpFailures...) + + return partialError("partial server info collection", failures, timeout) +} diff --git a/src/internal/app/collect_diagnostics_test.go b/src/internal/app/collect_diagnostics_test.go new file mode 100644 index 0000000..585bf0d --- /dev/null +++ b/src/internal/app/collect_diagnostics_test.go @@ -0,0 +1,440 @@ +package app + +import ( + "context" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +func TestCollectPlanOrdering(t *testing.T) { + iops := NewInfrahubOps() + plan := iops.collectPlan(CollectOptions{}) + + want := []string{ + "logs/infrahub-server", + "logs/task-worker", + "logs/database", + "logs/message-queue", + "logs/cache", + "logs/task-manager", + "logs/task-manager-db", + "logs/task-manager-background-svc", + "database-logs", + "message-queue-status", + "cache-status", + "task-worker-state", + "task-manager-state", + "server-info", + "metrics", + // Opt-in extras stay last: the benchmark generates load, and the + // include-backup collector may stop/restart application containers + // (inherited backup behavior), so backup stays last of all. + "benchmark", + "backup", + } + + if len(plan) != len(want) { + names := make([]string, len(plan)) + for i, c := range plan { + names[i] = c.name + } + t.Fatalf("plan has %d collectors %v, want %d %v", len(plan), names, len(want), want) + } + for i, name := range want { + if plan[i].name != name { + t.Errorf("plan[%d].name = %q, want %q", i, plan[i].name, name) + } + if plan[i].run == nil { + t.Errorf("plan[%d] (%s) has no run function", i, name) + } + } + + // Every service-bound collector must carry an absent-service skip + // precondition; metrics applies to the whole deployment and has none. + for i, c := range plan { + wantSkip := c.name != "metrics" + if (c.skip != nil) != wantSkip { + t.Errorf("plan[%d] (%s) skip precondition presence = %v, want %v", i, c.name, c.skip != nil, wantSkip) + } + } +} + +func TestSkipWhenServiceAbsent(t *testing.T) { + tests := []struct { + name string + cc *collectContext + service string + wantSkip bool + wantReason string + }{ + { + name: "service with replicas is not skipped", + cc: &collectContext{backend: newFakeCollectBackend()}, + service: "infrahub-server", + wantSkip: false, + wantReason: "", + }, + { + name: "service without replicas is skipped", + cc: &collectContext{backend: newFakeCollectBackend()}, + service: "task-manager-background-svc", + wantSkip: true, + wantReason: "service not deployed", + }, + { + name: "backend without collect primitives defers to run", + cc: &collectContext{backend: &bareBackend{name: "docker"}}, + service: "cache", + wantSkip: false, + wantReason: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + skip, reason := skipWhenServiceAbsent(tt.service)(tt.cc) + if skip != tt.wantSkip || reason != tt.wantReason { + t.Errorf("skipWhenServiceAbsent(%q) = (%v, %q), want (%v, %q)", + tt.service, skip, reason, tt.wantSkip, tt.wantReason) + } + }) + } +} + +func TestMessageQueueDumps(t *testing.T) { + dumps := messageQueueDumps() + + wantFiles := []string{ + "queues.txt", "exchanges.txt", "bindings.txt", + "connections.txt", "channels.txt", "status.txt", "environment.txt", + } + if len(dumps) != len(wantFiles) { + t.Fatalf("messageQueueDumps() has %d dumps, want %d", len(dumps), len(wantFiles)) + } + + masked := map[string]bool{"status.txt": true, "environment.txt": true} + for i, dump := range dumps { + if dump.filename != wantFiles[i] { + t.Errorf("dumps[%d].filename = %q, want %q", i, dump.filename, wantFiles[i]) + } + if dump.command[0] != "rabbitmqctl" { + t.Errorf("dumps[%d] (%s) command = %v, want a rabbitmqctl invocation", i, dump.filename, dump.command) + } + if masked[dump.filename] { + if dump.mask == nil { + t.Errorf("dumps[%d] (%s) has no mask, want Erlang-config masking (research R5)", i, dump.filename) + continue + } + got := dump.mask(`{default_pass,<<"guest">>}`) + if want := "{default_pass," + maskedValue + "}"; got != want { + t.Errorf("dumps[%d] (%s) mask output = %q, want %q", i, dump.filename, got, want) + } + } else if dump.mask != nil { + t.Errorf("dumps[%d] (%s) is masked, want raw output", i, dump.filename) + } + } +} + +func TestCacheDumps(t *testing.T) { + dumps := cacheDumps() + + wantFiles := []string{"info.txt", "clients.txt", "config.txt", "slowlog.txt", "dbsize.txt"} + if len(dumps) != len(wantFiles) { + t.Fatalf("cacheDumps() has %d dumps, want %d", len(dumps), len(wantFiles)) + } + + for i, dump := range dumps { + if dump.filename != wantFiles[i] { + t.Errorf("dumps[%d].filename = %q, want %q", i, dump.filename, wantFiles[i]) + } + if dump.command[0] != "redis-cli" { + t.Errorf("dumps[%d] (%s) command = %v, want a redis-cli invocation", i, dump.filename, dump.command) + } + if dump.filename == "config.txt" { + if dump.mask == nil { + t.Errorf("config.txt has no mask, want key/value-pair masking (research R5)") + continue + } + got := dump.mask("requirepass\nhunter2") + if want := "requirepass\n" + maskedValue; got != want { + t.Errorf("config.txt mask output = %q, want %q", got, want) + } + } else if dump.mask != nil { + t.Errorf("dumps[%d] (%s) is masked, want raw output", i, dump.filename) + } + } +} + +func TestTaskWorkerDumps(t *testing.T) { + dumps := taskWorkerDumps() + + wantFiles := []string{"version.txt", "config.txt", "work-pools.txt"} + if len(dumps) != len(wantFiles) { + t.Fatalf("taskWorkerDumps() has %d dumps, want %d", len(dumps), len(wantFiles)) + } + + for i, dump := range dumps { + if dump.filename != wantFiles[i] { + t.Errorf("dumps[%d].filename = %q, want %q", i, dump.filename, wantFiles[i]) + } + if dump.command[0] != "prefect" { + t.Errorf("dumps[%d] (%s) command = %v, want a prefect invocation", i, dump.filename, dump.command) + } + } + + // The worker configuration can carry Prefect API credentials. + config := dumps[1] + if config.mask == nil { + t.Fatal("config.txt has no mask, want env-style masking (research R5)") + } + got := config.mask("PREFECT_API_KEY='abc123'") + if want := "PREFECT_API_KEY=" + maskedValue; got != want { + t.Errorf("config.txt mask output = %q, want %q", got, want) + } +} + +func TestPrefectServerCommand(t *testing.T) { + cmd := prefectServerCommand("prefect", "work-pool", "ls") + + if len(cmd) != 7 { + t.Fatalf("prefectServerCommand produced %d elements %v, want 7", len(cmd), cmd) + } + if cmd[0] != "sh" || cmd[1] != "-c" { + t.Errorf("command prefix = %v, want a sh -c wrapper", cmd[:2]) + } + script := cmd[2] + if !strings.Contains(script, `${PREFECT_API_URL:-`+prefectServerAPIFallback+`}`) { + t.Errorf("wrapper script %q does not default PREFECT_API_URL to %s", script, prefectServerAPIFallback) + } + if !strings.Contains(script, `exec "$@"`) { + t.Errorf("wrapper script %q does not exec the wrapped command", script) + } + wantTail := []string{"sh", "prefect", "work-pool", "ls"} + for i, arg := range wantTail { + if cmd[3+i] != arg { + t.Errorf("cmd[%d] = %q, want %q", 3+i, cmd[3+i], arg) + } + } +} + +func TestTaskManagerDumps(t *testing.T) { + dumps := taskManagerDumps() + + wantFiles := []string{"work-pools.txt", "work-queues.txt", "flow-runs.txt", "automations.txt"} + if len(dumps) != len(wantFiles) { + t.Fatalf("taskManagerDumps() has %d dumps, want %d", len(dumps), len(wantFiles)) + } + + for i, dump := range dumps { + if dump.filename != wantFiles[i] { + t.Errorf("dumps[%d].filename = %q, want %q", i, dump.filename, wantFiles[i]) + } + if dump.command[0] != "sh" || !contains(dump.command, "prefect") { + t.Errorf("dumps[%d] (%s) command = %v, want a wrapped prefect invocation", i, dump.filename, dump.command) + } + } +} + +func TestServerAPITargets(t *testing.T) { + targets := serverAPITargets() + + want := []struct { + filename string + path string + masked bool + }{ + {"info.json", "/api/info", false}, + {"config.json", "/api/config", true}, + {"schema.json", "/api/schema", false}, + } + if len(targets) != len(want) { + t.Fatalf("serverAPITargets() has %d targets, want %d", len(targets), len(want)) + } + + for i, tt := range want { + target := targets[i] + if target.filename != tt.filename || target.path != tt.path { + t.Errorf("targets[%d] = {%q %q}, want {%q %q}", i, target.filename, target.path, tt.filename, tt.path) + } + if (target.mask != nil) != tt.masked { + t.Errorf("targets[%d] (%s) mask presence = %v, want %v", i, tt.filename, target.mask != nil, tt.masked) + continue + } + if tt.masked { + got := target.mask(`{"security":{"secret_key":"abc123"}}`) + if strings.Contains(got, "abc123") || !strings.Contains(got, maskedValue) { + t.Errorf("config mask output %q still leaks the secret", got) + } + } + } +} + +func TestServerAPIFetchCommand(t *testing.T) { + url := "http://infrahub-server:8000/api/config" + cmd := serverAPIFetchCommand(url) + + if len(cmd) != 4 || cmd[0] != "python" || cmd[1] != "-c" { + t.Fatalf("serverAPIFetchCommand = %v, want a python -c invocation with the URL argument", cmd) + } + if cmd[3] != url { + t.Errorf("URL argument = %q, want %q", cmd[3], url) + } + script := cmd[2] + for _, fragment := range []string{"httpx", "sys.argv[1]", "resp.status_code >= 400", "sys.exit(1)"} { + if !strings.Contains(script, fragment) { + t.Errorf("fetch script missing %q:\n%s", fragment, script) + } + } +} + +// timeoutExecBackend is a collect backend whose bounded exec always times out, +// used to prove a command timeout survives a real aggregating collector's +// string aggregation and reaches the manifest as the bare contract reason +// (FIX-4). +type timeoutExecBackend struct { + fakeCollectBackend + timeout time.Duration +} + +func (b *timeoutExecBackend) ExecContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) (string, error) { + return "", &timeoutError{timeout: b.timeout} +} + +var _ contextExecer = (*timeoutExecBackend)(nil) + +// TestAggregatingCollector_TimeoutReasonSurvivesAggregation drives the real +// message-queue collector (execDumpCollector → runExecDumps → execDumpsInto) +// against a backend whose exec times out, and asserts the manifest reason is +// exactly the CLI contract's bare "timed out after 60s" — not a verbose +// composite (FIX-4). +func TestAggregatingCollector_TimeoutReasonSurvivesAggregation(t *testing.T) { + iops := NewInfrahubOps() + backend := &timeoutExecBackend{ + fakeCollectBackend: *newFakeCollectBackend(), + timeout: collectExecTimeout, + } + // message-queue must have a replica so the collector runs instead of being + // skipped as not deployed. + backend.replicas["message-queue"] = []Replica{{Service: "message-queue", Container: "message-queue-1"}} + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + if err := iops.runCollectPlan(backend, opts, []collector{messageQueueCollector()}); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + if len(manifest.Collectors) != 1 { + t.Fatalf("manifest has %d entries %+v, want 1", len(manifest.Collectors), manifest.Collectors) + } + want := CollectorResult{Name: "message-queue-status", Status: collectorStatusFailed, Reason: "timed out after 60s"} + if manifest.Collectors[0] != want { + t.Errorf("collector result = %+v, want %+v (bare timeout reason per contracts/cli.md, FIX-4)", manifest.Collectors[0], want) + } +} + +// copyRecordingBackend records the source paths passed to CopyFromContext and +// scripts the Neo4j log-directory listing, exercising collectDatabaseLogs +// without a real container (FIX-T2, FIX-7). +type copyRecordingBackend struct { + bareBackend + copied []string + logFiles string +} + +func (b *copyRecordingBackend) CopyFromContext(ctx context.Context, timeout time.Duration, service, src, dest string) error { + b.copied = append(b.copied, src) + return nil +} + +func (b *copyRecordingBackend) ExecContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) (string, error) { + return b.logFiles, nil +} + +var ( + _ contextCopier = (*copyRecordingBackend)(nil) + _ contextExecer = (*copyRecordingBackend)(nil) +) + +// TestCollectDatabaseLogs_IncludeQueries covers the FR-004 --include-queries +// path: the default run copies exactly neo4j.log+debug.log, --include-queries +// copies the live-enumerated set including query logs, and an empty enumeration +// falls back to the default set rather than recording an empty success (FIX-7). +func TestCollectDatabaseLogs_IncludeQueries(t *testing.T) { + newCC := func(backend EnvironmentBackend, opts CollectOptions) *collectContext { + return &collectContext{ctx: context.Background(), backend: backend, bundleDir: t.TempDir(), opts: opts} + } + + t.Run("default copies only the Neo4j server logs", func(t *testing.T) { + backend := ©RecordingBackend{bareBackend: bareBackend{name: "docker"}} + if err := collectDatabaseLogs(newCC(backend, CollectOptions{})); err != nil { + t.Fatalf("collectDatabaseLogs failed: %v", err) + } + want := []string{"/logs/neo4j.log", "/logs/debug.log"} + if !reflect.DeepEqual(backend.copied, want) { + t.Errorf("copied = %v, want %v", backend.copied, want) + } + }) + + t.Run("include-queries copies the enumerated set including query logs", func(t *testing.T) { + backend := ©RecordingBackend{ + bareBackend: bareBackend{name: "docker"}, + logFiles: "neo4j.log\ndebug.log\nquery.log\nquery.log.1\n", + } + if err := collectDatabaseLogs(newCC(backend, CollectOptions{IncludeQueries: true})); err != nil { + t.Fatalf("collectDatabaseLogs failed: %v", err) + } + want := []string{"/logs/neo4j.log", "/logs/debug.log", "/logs/query.log", "/logs/query.log.1"} + if !reflect.DeepEqual(backend.copied, want) { + t.Errorf("copied = %v, want %v", backend.copied, want) + } + }) + + t.Run("empty enumeration falls back to the default set (FIX-7)", func(t *testing.T) { + backend := ©RecordingBackend{bareBackend: bareBackend{name: "docker"}, logFiles: ""} + if err := collectDatabaseLogs(newCC(backend, CollectOptions{IncludeQueries: true})); err != nil { + t.Fatalf("collectDatabaseLogs failed: %v", err) + } + want := []string{"/logs/neo4j.log", "/logs/debug.log"} + if !reflect.DeepEqual(backend.copied, want) { + t.Errorf("copied = %v, want %v (empty enumeration must fall back, not record an empty success)", backend.copied, want) + } + }) +} + +func TestReplicaBaseName(t *testing.T) { + tests := []struct { + name string + replica Replica + multiContainer bool + want string + }{ + { + name: "docker container name", + replica: Replica{Service: "task-worker", Container: "infrahub-task-worker-1"}, + want: "infrahub-task-worker-1", + }, + { + name: "kubernetes single-container pod", + replica: Replica{Service: "task-worker", Pod: "task-worker-abc", Container: "task-worker"}, + want: "task-worker-abc", + }, + { + name: "kubernetes multi-container pod", + replica: Replica{Service: "database", Pod: "database-0", Container: "sidecar"}, + multiContainer: true, + want: "database-0_sidecar", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := replicaBaseName(tt.replica, tt.multiContainer); got != tt.want { + t.Errorf("replicaBaseName(%+v, %v) = %q, want %q", tt.replica, tt.multiContainer, got, tt.want) + } + }) + } +} diff --git a/src/internal/app/collect_extras.go b/src/internal/app/collect_extras.go new file mode 100644 index 0000000..d0ca203 --- /dev/null +++ b/src/internal/app/collect_extras.go @@ -0,0 +1,297 @@ +package app + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/sirupsen/logrus" +) + +// backupCollectorName is the manifest identifier of the include-backup +// collector (specs/003-collect-tool/data-model.md, CollectorResult.Name). +const backupCollectorName = "backup" + +// backupArchiveGlob matches the tarball archives CreateBackup writes into the +// backup directory (see generateBackupFilename). +const backupArchiveGlob = "infrahub_backup_*.tar.gz" + +// backupRunner produces a backup and returns the path of the created archive +// (empty when the artifact cannot be located). The production runner is +// runStandardBackup; unit tests inject fakes. +type backupRunner func(cc *collectContext) (string, error) + +// includeBackupCollector returns the opt-in --include-backup collector +// (FR-014, research R10). It must stay last in the run plan: the delegated +// backup inherits the standard backup behavior, which may stop and restart +// application containers, so every read-only collector stages its files +// first and a backup failure cannot taint the diagnostics (US3 scenario 2). +// The produced archive stays a standalone file in the standard backup +// directory — referenced by the manifest, not embedded in the bundle. +func includeBackupCollector(run backupRunner) collector { + return collector{ + name: backupCollectorName, + skip: func(cc *collectContext) (bool, string) { + return !cc.opts.IncludeBackup, "not requested" + }, + run: func(cc *collectContext) error { + artifact, err := run(cc) + if err != nil { + return fmt.Errorf("backup failed: %w", err) + } + cc.setArtifact(artifact) + return nil + }, + } +} + +// runStandardBackup delegates to the existing CreateBackup: force=true, all +// Neo4j metadata, task-manager database included, no S3 upload, no sleep, no +// redaction, no encryption (research R10). CreateBackup does not return the +// path it wrote, so the new archive is identified by diffing the backup +// directory listing around the call. +// +// force=true is deliberate. Without it, CreateBackup runs waitForRunningTasks, +// an unbounded loop that only returns once zero Prefect tasks are +// running/pending. A troubleshooting bundle is collected non-interactively +// against a live — often busy — instance whose recurring background tasks and +// automations never fully drain (this is why every collector is +// timeout-bounded), so the wait would hang or, on failure, abort the backup +// and leave no artifact. --force skips only that consistency gate; it is how +// backups are taken non-interactively (the backup e2e suite uses it) and does +// not weaken the backup's own integrity guarantees (metadata + checksums). +func runStandardBackup(cc *collectContext) (string, error) { + backupDir := cc.iops.config.BackupDir + + before, err := listBackupArchives(backupDir) + if err != nil { + return "", err + } + + if err := cc.iops.CreateBackup(true, "all", false, false, false, 0, false, false, ""); err != nil { + return "", err + } + + after, err := listBackupArchives(backupDir) + if err != nil { + return "", err + } + + artifact := newestNewArchive(before, after) + if artifact == "" { + // Nothing to reference (e.g. a non-tarball backend wrote elsewhere). + // The backup itself succeeded, so this is a warning, not a failure. + logrus.Warnf("Backup completed but no new archive was found in %s; the manifest entry will carry no artifact path", backupDir) + return "", nil + } + + if abs, absErr := filepath.Abs(artifact); absErr == nil { + artifact = abs + } + return artifact, nil +} + +// listBackupArchives returns the backup archives currently present in dir. +// A missing directory yields an empty list (CreateBackup creates it). +func listBackupArchives(dir string) ([]string, error) { + archives, err := filepath.Glob(filepath.Join(dir, backupArchiveGlob)) + if err != nil { + return nil, fmt.Errorf("failed to list backup archives in %s: %w", dir, err) + } + return archives, nil +} + +// newestNewArchive returns the newest archive present in after but absent +// from before. Archive names embed a YYYYMMDD_HHMMSS timestamp, so within one +// directory the lexically greatest new path is the most recent one. +func newestNewArchive(before, after []string) string { + seen := make(map[string]struct{}, len(before)) + for _, path := range before { + seen[path] = struct{}{} + } + + newest := "" + for _, path := range after { + if _, ok := seen[path]; ok { + continue + } + if path > newest { + newest = path + } + } + return newest +} + +// --- benchmark (T036, FR-013, research R11) --- + +// benchmarkCollectorName is the manifest identifier of the opt-in --benchmark +// collector (specs/003-collect-tool/data-model.md, CollectorResult.Name). +const benchmarkCollectorName = "benchmark" + +// benchmarkBundleDir is the bundle directory the benchmark collector owns +// (contracts/bundle-layout.md: present only when --benchmark ran). +const benchmarkBundleDir = "benchmark" + +// defaultBenchmarkImage is the OpsMill benchmark image, ported from the Python +// `invoke bundle collect` implementation (opsmill/infrahub +// tasks/container_ops.py, collect_benchmark: `docker run --pull always --rm +// registry.opsmill.io/opsmill/bench`). The image measures the host against +// Infrahub's hardware requirements: disk read/write IOPS (fio), total memory, +// and single-core CPU performance. +const defaultBenchmarkImage = "registry.opsmill.io/opsmill/bench" + +// benchmarkImageEnvVar overrides the benchmark image reference, keeping +// air-gapped-with-private-registry workflows possible (research R11). +const benchmarkImageEnvVar = "INFRAHUB_BENCHMARK_IMAGE" + +// resolveBenchmarkImage returns the effective benchmark image reference: +// the INFRAHUB_BENCHMARK_IMAGE override when set, the build-time default +// otherwise. getenv is injectable for tests; production passes os.Getenv. +func resolveBenchmarkImage(getenv func(string) string) string { + if image := getenv(benchmarkImageEnvVar); image != "" { + return image + } + return defaultBenchmarkImage +} + +// benchmarkResourceName names the transient benchmark container/pod after the +// collect run. Pod names must be valid RFC 1123 labels, so the collect ID is +// lowercased and its underscore replaced. +func benchmarkResourceName(collectID string) string { + return "infrahub-collect-benchmark-" + strings.ReplaceAll(strings.ToLower(collectID), "_", "-") +} + +// benchmarkRunner executes the benchmark image against the active environment +// and returns its raw combined output. The production runner is +// runEnvironmentBenchmark; unit tests inject fakes. +type benchmarkRunner func(cc *collectContext, image string) (string, error) + +// benchmarkCollector returns the opt-in --benchmark collector (FR-013, +// research R11). The benchmark image download is the only network egress the +// tool ever performs, and only under this flag. A failing image pull or run +// is recorded as skipped with a warning — never failed — so air-gapped runs +// stay clean (spec air-gapped edge case); only a local staging error marks +// the collector failed. +func benchmarkCollector(run benchmarkRunner) collector { + return collector{ + name: benchmarkCollectorName, + skip: func(cc *collectContext) (bool, string) { + return !cc.opts.Benchmark, "not requested" + }, + run: func(cc *collectContext) error { + image := resolveBenchmarkImage(os.Getenv) + logrus.Infof("Running benchmark image %s (requires image download)", image) + output, err := run(cc, image) + if err != nil { + return &collectSkipError{ + reason: fmt.Sprintf("benchmark image %s could not be pulled or run: %v", image, err), + } + } + return stageBenchmarkOutput(cc.bundleDir, output) + }, + } +} + +// runEnvironmentBenchmark dispatches the benchmark to the active backend: a +// transient `docker run` attached to the compose project's network, or a +// one-off attached pod in the Kubernetes namespace. Both backends delete the +// container/pod they created — permitted under FR-010, which protects the +// deployment's workloads: the benchmark resource is the tool's own. +func runEnvironmentBenchmark(cc *collectContext, image string) (string, error) { + name := benchmarkResourceName(cc.manifest.CollectID) + switch backend := cc.backend.(type) { + case *DockerBackend: + return backend.RunBenchmark(cc.ctx, image, name) + case *KubernetesBackend: + return backend.RunBenchmark(cc.ctx, image, name) + default: + return "", fmt.Errorf("%s environment does not support the benchmark", cc.backend.Name()) + } +} + +// commandErrorLine condenses a failed command's combined output into its one +// actionable line: the last line mentioning an error (docker and kubectl both +// print one), falling back to the last non-empty line. This keeps manifest +// reasons readable without the image pull-progress noise. +func commandErrorLine(output string) string { + lines := nonEmptyLines(output) + if len(lines) == 0 { + return "" + } + for i := len(lines) - 1; i >= 0; i-- { + if strings.Contains(strings.ToLower(lines[i]), "error") { + return lines[i] + } + } + return lines[len(lines)-1] +} + +// benchmarkResultRe extracts one benchmark result line +// (": [ MB] - Required: [ MB] : "), +// ported from the Python collect_benchmark parser. +var benchmarkResultRe = regexp.MustCompile(`(\w+(?: \w+)*): (\d+)(?: MB)? - Required: (\d+)(?: MB)? : (\w+)`) + +// ansiEscapeRe matches the ANSI SGR color sequences the benchmark entrypoint +// wraps its result lines in. +var ansiEscapeRe = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +// benchmarkMeasure is one parsed benchmark result; the field names mirror the +// Python tool's benchmark.json entries. +type benchmarkMeasure struct { + Value int `json:"value"` + Required int `json:"required"` + Status string `json:"status"` +} + +// benchmarkReport is the benchmark.json document, shape-compatible with the +// Python tool's output. +type benchmarkReport struct { + RawOutput string `json:"raw_output"` + Results map[string]benchmarkMeasure `json:"results"` +} + +// parseBenchmarkResults strips color codes from the benchmark output and +// extracts the result lines into keyed measures ("memory", "disk_read_iops", +// ...). It returns the cleaned output and the measures; unmatched output +// yields an empty map, never an error — the raw log is authoritative. +func parseBenchmarkResults(output string) (string, map[string]benchmarkMeasure) { + clean := ansiEscapeRe.ReplaceAllString(output, "") + results := map[string]benchmarkMeasure{} + for _, match := range benchmarkResultRe.FindAllStringSubmatch(clean, -1) { + value, err := strconv.Atoi(match[2]) + if err != nil { + continue + } + required, err := strconv.Atoi(match[3]) + if err != nil { + continue + } + key := strings.ToLower(strings.ReplaceAll(match[1], " ", "_")) + results[key] = benchmarkMeasure{Value: value, Required: required, Status: match[4]} + } + return clean, results +} + +// stageBenchmarkOutput writes the raw benchmark output (benchmark.log) and +// the parsed results (benchmark.json) into bundle/benchmark/. +func stageBenchmarkOutput(bundleDir, output string) error { + dir := filepath.Join(bundleDir, benchmarkBundleDir) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create %s directory: %w", benchmarkBundleDir, err) + } + + if err := writeDumpFile(filepath.Join(dir, "benchmark.log"), output, nil); err != nil { + return err + } + + raw, results := parseBenchmarkResults(output) + data, err := json.MarshalIndent(benchmarkReport{RawOutput: raw, Results: results}, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal benchmark results: %w", err) + } + return writeDumpFile(filepath.Join(dir, "benchmark.json"), string(data), nil) +} diff --git a/src/internal/app/collect_extras_test.go b/src/internal/app/collect_extras_test.go new file mode 100644 index 0000000..b7a42e4 --- /dev/null +++ b/src/internal/app/collect_extras_test.go @@ -0,0 +1,472 @@ +package app + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// TestIncludeBackupCollector_OutcomeRecording drives the include-backup +// collector through runCollectPlan with fake backup runners (T034): success +// records the artifact path, failure records a failed entry while the bundle +// is still produced (US3 scenario 2), and an unset flag records skipped. +func TestIncludeBackupCollector_OutcomeRecording(t *testing.T) { + t.Run("success records the backup artifact", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000, IncludeBackup: true} + + const artifact = "/backups/infrahub_backup_20260703_101530.tar.gz" + ran := false + plan := []collector{includeBackupCollector(func(cc *collectContext) (string, error) { + ran = true + return artifact, nil + })} + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + if !ran { + t.Fatal("backup runner was not invoked despite --include-backup") + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + want := CollectorResult{Name: "backup", Status: collectorStatusSuccess, Artifact: artifact} + if len(manifest.Collectors) != 1 || manifest.Collectors[0] != want { + t.Errorf("collectors = %+v, want [%+v]", manifest.Collectors, want) + } + }) + + t.Run("backup failure records failed and still produces the bundle", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000, IncludeBackup: true} + + plan := []collector{ + { + name: "metrics", + run: func(cc *collectContext) error { return nil }, + }, + includeBackupCollector(func(cc *collectContext) (string, error) { + return "", fmt.Errorf("database container unreachable") + }), + } + + // US3 scenario 2: a backup failure never aborts the run. + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed after backup error, want bundle still produced: %v", err) + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + wantResults := []CollectorResult{ + {Name: "metrics", Status: collectorStatusSuccess}, + {Name: "backup", Status: collectorStatusFailed, Reason: "backup failed: database container unreachable"}, + } + if len(manifest.Collectors) != len(wantResults) { + t.Fatalf("manifest has %d collector entries %+v, want %d", len(manifest.Collectors), manifest.Collectors, len(wantResults)) + } + for i, want := range wantResults { + if manifest.Collectors[i] != want { + t.Errorf("collectors[%d] = %+v, want %+v", i, manifest.Collectors[i], want) + } + } + }) + + t.Run("skipped as not requested when the flag is unset", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000, IncludeBackup: false} + + plan := []collector{includeBackupCollector(func(cc *collectContext) (string, error) { + t.Error("backup runner invoked without --include-backup") + return "", nil + })} + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + want := CollectorResult{Name: "backup", Status: collectorStatusSkipped, Reason: "not requested"} + if len(manifest.Collectors) != 1 || manifest.Collectors[0] != want { + t.Errorf("collectors = %+v, want [%+v]", manifest.Collectors, want) + } + }) +} + +// TestCollectorResult_ArtifactJSONShape pins the manifest contract for the +// artifact reference (contracts/manifest.schema.json): the field serializes +// as "artifact" and is omitted from entries that did not produce one. +func TestCollectorResult_ArtifactJSONShape(t *testing.T) { + manifest := newBundleManifest(generateCollectID(), "docker", 100000) + manifest.recordSuccessArtifact("backup", "/backups/infrahub_backup_20260703_101530.tar.gz") + manifest.recordSuccess("metrics") + + data, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + var decoded map[string]any + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + collectors, ok := decoded["collectors"].([]any) + if !ok || len(collectors) != 2 { + t.Fatalf("collectors = %v, want array of 2 entries", decoded["collectors"]) + } + + backupEntry, ok := collectors[0].(map[string]any) + if !ok { + t.Fatalf("collectors[0] = %T, want object", collectors[0]) + } + if backupEntry["artifact"] != "/backups/infrahub_backup_20260703_101530.tar.gz" { + t.Errorf("backup entry artifact = %v, want the archive path", backupEntry["artifact"]) + } + + metricsEntry, ok := collectors[1].(map[string]any) + if !ok { + t.Fatalf("collectors[1] = %T, want object", collectors[1]) + } + if _, present := metricsEntry["artifact"]; present { + t.Errorf("metrics entry = %v, want no artifact field (omitempty)", metricsEntry) + } +} + +// sampleBenchmarkOutput mimics the OpsMill bench image's entrypoint output, +// including the ANSI color codes it wraps result lines in. +const sampleBenchmarkOutput = "Running Disk IOPS benchmark... hold on\n" + + "Benchmark results:\n" + + "\x1b[0;32mMemory: 15895 MB - Required: 7000 MB : OK\n" + + "\x1b[0;32mCPU Single Core Perf: 3500 - Required: 2000 : OK\n" + + "\x1b[0;31mDisk Read IOPS: 4231 - Required: 5000 : KO\n" + + "\x1b[0;32mDisk Write IOPS: 5120 - Required: 5000 : OK\n" + + "\x1b[0m\n" + + "Benchmark completed...\n" + +// TestBenchmarkCollector_OutcomeRecording drives the benchmark collector +// through runCollectPlan with fake runners (T036): an unset flag records +// skipped/not requested, success stages bundle/benchmark/ files, and an image +// pull/run failure records skipped with a warning reason — never failed +// (FR-013, research R11, spec air-gapped edge case). +func TestBenchmarkCollector_OutcomeRecording(t *testing.T) { + t.Run("skipped as not requested when the flag is unset", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000, Benchmark: false} + + plan := []collector{benchmarkCollector(func(cc *collectContext, image string) (string, error) { + t.Error("benchmark runner invoked without --benchmark") + return "", nil + })} + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + want := CollectorResult{Name: "benchmark", Status: collectorStatusSkipped, Reason: "not requested"} + if len(manifest.Collectors) != 1 || manifest.Collectors[0] != want { + t.Errorf("collectors = %+v, want [%+v]", manifest.Collectors, want) + } + }) + + t.Run("success stages benchmark.log and benchmark.json", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000, Benchmark: true} + + // The override must reach the runner (INFRAHUB_BENCHMARK_IMAGE seam). + t.Setenv(benchmarkImageEnvVar, "registry.example.com/opsmill/bench:pinned") + gotImage := "" + plan := []collector{benchmarkCollector(func(cc *collectContext, image string) (string, error) { + gotImage = image + return sampleBenchmarkOutput, nil + })} + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + if gotImage != "registry.example.com/opsmill/bench:pinned" { + t.Errorf("runner image = %q, want the INFRAHUB_BENCHMARK_IMAGE override", gotImage) + } + + manifest, destDir := extractBundleManifest(t, findArchive(t, outputDir)) + want := CollectorResult{Name: "benchmark", Status: collectorStatusSuccess} + if len(manifest.Collectors) != 1 || manifest.Collectors[0] != want { + t.Errorf("collectors = %+v, want [%+v]", manifest.Collectors, want) + } + + logData, err := os.ReadFile(filepath.Join(destDir, "bundle", "benchmark", "benchmark.log")) + if err != nil { + t.Fatalf("benchmark/benchmark.log missing from archive: %v", err) + } + if string(logData) != sampleBenchmarkOutput { + t.Errorf("benchmark.log = %q, want the raw runner output", logData) + } + + jsonData, err := os.ReadFile(filepath.Join(destDir, "bundle", "benchmark", "benchmark.json")) + if err != nil { + t.Fatalf("benchmark/benchmark.json missing from archive: %v", err) + } + var report benchmarkReport + if err := json.Unmarshal(jsonData, &report); err != nil { + t.Fatalf("benchmark.json is not valid JSON: %v", err) + } + if want := (benchmarkMeasure{Value: 4231, Required: 5000, Status: "KO"}); report.Results["disk_read_iops"] != want { + t.Errorf("disk_read_iops = %+v, want %+v", report.Results["disk_read_iops"], want) + } + if len(report.Results) != 4 { + t.Errorf("parsed %d results %+v, want 4", len(report.Results), report.Results) + } + }) + + t.Run("image pull or run failure records skipped with a warning, not failed", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000, Benchmark: true} + t.Setenv(benchmarkImageEnvVar, "") // pin the default image for the reason assertion + + plan := []collector{ + benchmarkCollector(func(cc *collectContext, image string) (string, error) { + return "", fmt.Errorf("docker run failed: manifest unknown") + }), + { + name: "metrics", + run: func(cc *collectContext) error { return nil }, + }, + } + + // Air-gapped edge case: the run continues and still exits 0. + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed after benchmark error, want bundle still produced: %v", err) + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + wantResults := []CollectorResult{ + { + Name: "benchmark", + Status: collectorStatusSkipped, + Reason: "benchmark image " + defaultBenchmarkImage + " could not be pulled or run: docker run failed: manifest unknown", + }, + {Name: "metrics", Status: collectorStatusSuccess}, + } + if len(manifest.Collectors) != len(wantResults) { + t.Fatalf("manifest has %d collector entries %+v, want %d", len(manifest.Collectors), manifest.Collectors, len(wantResults)) + } + for i, want := range wantResults { + if manifest.Collectors[i] != want { + t.Errorf("collectors[%d] = %+v, want %+v", i, manifest.Collectors[i], want) + } + } + }) +} + +func TestResolveBenchmarkImage(t *testing.T) { + env := map[string]string{} + getenv := func(key string) string { return env[key] } + + if got := resolveBenchmarkImage(getenv); got != defaultBenchmarkImage { + t.Errorf("resolveBenchmarkImage() = %q, want the default %q", got, defaultBenchmarkImage) + } + + env[benchmarkImageEnvVar] = "registry.example.com/opsmill/bench:pinned" + if got := resolveBenchmarkImage(getenv); got != "registry.example.com/opsmill/bench:pinned" { + t.Errorf("resolveBenchmarkImage() = %q, want the override", got) + } +} + +func TestBenchmarkResourceName(t *testing.T) { + got := benchmarkResourceName("20260703_141530") + want := "infrahub-collect-benchmark-20260703-141530" + if got != want { + t.Errorf("benchmarkResourceName() = %q, want %q (RFC 1123 pod-name safe)", got, want) + } +} + +func TestDockerBenchmarkRunArgs(t *testing.T) { + name := "infrahub-collect-benchmark-20260703-141530" + + withNetwork := dockerBenchmarkRunArgs(name, "infrahub_default", defaultBenchmarkImage) + want := []string{ + "run", "--pull", "always", "--rm", "--name", name, + "--network", "infrahub_default", defaultBenchmarkImage, + } + if !reflect.DeepEqual(withNetwork, want) { + t.Errorf("dockerBenchmarkRunArgs(network) = %v, want %v", withNetwork, want) + } + + withoutNetwork := dockerBenchmarkRunArgs(name, "", defaultBenchmarkImage) + want = []string{"run", "--pull", "always", "--rm", "--name", name, defaultBenchmarkImage} + if !reflect.DeepEqual(withoutNetwork, want) { + t.Errorf("dockerBenchmarkRunArgs(no network) = %v, want %v", withoutNetwork, want) + } +} + +func TestKubectlBenchmarkRunArgs(t *testing.T) { + got := kubectlBenchmarkRunArgs("infrahub", "infrahub-collect-benchmark-20260703-141530", defaultBenchmarkImage) + want := []string{ + "run", "infrahub-collect-benchmark-20260703-141530", + "-n", "infrahub", + "--image=" + defaultBenchmarkImage, + "--image-pull-policy=Always", + "--restart=Never", + "--attach", + "--rm", + "--quiet", + "--pod-running-timeout=5m", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("kubectlBenchmarkRunArgs() = %v, want %v", got, want) + } +} + +func TestPickComposeNetwork(t *testing.T) { + tests := []struct { + name string + project string + networks []string + want string + }{ + { + name: "prefers the compose default network", + project: "infrahub", + networks: []string{"infrahub_backendnet", "infrahub_default"}, + want: "infrahub_default", + }, + { + name: "falls back to the first project network in lexical order", + project: "infrahub", + networks: []string{"infrahub_frontnet", "infrahub_backendnet"}, + want: "infrahub_backendnet", + }, + { + name: "no networks means no attachment", + project: "infrahub", + networks: nil, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := pickComposeNetwork(tt.project, tt.networks); got != tt.want { + t.Errorf("pickComposeNetwork(%q, %v) = %q, want %q", tt.project, tt.networks, got, tt.want) + } + }) + } +} + +func TestCommandErrorLine(t *testing.T) { + tests := []struct { + name string + output string + want string + }{ + { + name: "picks the error line over docker's trailing help hint", + output: "Unable to find image 'x:latest' locally\n" + + "docker: Error response from daemon: manifest unknown\n" + + "Run 'docker run --help' for more information\n", + want: "docker: Error response from daemon: manifest unknown", + }, + { + name: "falls back to the last non-empty line", + output: "pulling layer 1/3\npulling layer 2/3\n\n", + want: "pulling layer 2/3", + }, + { + name: "empty output yields empty string", + output: " \n\n", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := commandErrorLine(tt.output); got != tt.want { + t.Errorf("commandErrorLine(%q) = %q, want %q", tt.output, got, tt.want) + } + }) + } +} + +func TestParseBenchmarkResults(t *testing.T) { + clean, results := parseBenchmarkResults(sampleBenchmarkOutput) + + if strings.Contains(clean, "\x1b[") { + t.Errorf("cleaned output still contains ANSI escapes: %q", clean) + } + + want := map[string]benchmarkMeasure{ + "memory": {Value: 15895, Required: 7000, Status: "OK"}, + "cpu_single_core_perf": {Value: 3500, Required: 2000, Status: "OK"}, + "disk_read_iops": {Value: 4231, Required: 5000, Status: "KO"}, + "disk_write_iops": {Value: 5120, Required: 5000, Status: "OK"}, + } + if !reflect.DeepEqual(results, want) { + t.Errorf("parseBenchmarkResults() = %+v, want %+v", results, want) + } + + if _, empty := parseBenchmarkResults("no results here"); len(empty) != 0 { + t.Errorf("parseBenchmarkResults(no results) = %+v, want empty", empty) + } +} + +func TestNewestNewArchive(t *testing.T) { + tests := []struct { + name string + before []string + after []string + want string + }{ + { + name: "single new archive", + before: []string{"/b/infrahub_backup_20260101_000000.tar.gz"}, + after: []string{ + "/b/infrahub_backup_20260101_000000.tar.gz", + "/b/infrahub_backup_20260703_101530.tar.gz", + }, + want: "/b/infrahub_backup_20260703_101530.tar.gz", + }, + { + name: "no new archive", + before: []string{"/b/infrahub_backup_20260101_000000.tar.gz"}, + after: []string{"/b/infrahub_backup_20260101_000000.tar.gz"}, + want: "", + }, + { + name: "empty directory before and after", + before: nil, + after: nil, + want: "", + }, + { + name: "several new archives pick the newest", + before: nil, + after: []string{ + "/b/infrahub_backup_20260703_101530.tar.gz", + "/b/infrahub_backup_20260703_101531.tar.gz", + }, + want: "/b/infrahub_backup_20260703_101531.tar.gz", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := newestNewArchive(tt.before, tt.after); got != tt.want { + t.Errorf("newestNewArchive(%v, %v) = %q, want %q", tt.before, tt.after, got, tt.want) + } + }) + } +} diff --git a/src/internal/app/collect_logs.go b/src/internal/app/collect_logs.go new file mode 100644 index 0000000..50ddecf --- /dev/null +++ b/src/internal/app/collect_logs.go @@ -0,0 +1,203 @@ +package app + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +// collectServices is the canonical Infrahub service list (the constitution's +// public deployment contract) in the order log collection walks it. +var collectServices = []string{ + "infrahub-server", + "task-worker", + "database", + "message-queue", + "cache", + "task-manager", + "task-manager-db", + "task-manager-background-svc", +} + +// serviceLogCollectors returns one collector per canonical service, each +// writing per-replica log files under bundle/logs// and recording a +// manifest entry named logs/. The run plan (T025) consumes this +// slice unchanged. +func serviceLogCollectors() []collector { + collectors := make([]collector, 0, len(collectServices)) + for _, service := range collectServices { + collectors = append(collectors, serviceLogCollector(service)) + } + return collectors +} + +// serviceLogCollector builds the log collector for one service. A service +// with no replicas (not deployed in this environment) is skipped; enumeration +// errors are left to run() so they surface as failed with a reason. +func serviceLogCollector(service string) collector { + return collector{ + name: "logs/" + service, + skip: func(cc *collectContext) (bool, string) { + cb, err := cc.collect() + if err != nil { + return false, "" // run() reports the unsupported backend as failed + } + replicas, err := cb.ServiceReplicas(service) + if err != nil { + return false, "" // run() reports the enumeration error as failed + } + if len(replicas) == 0 { + return true, "service not deployed" + } + return false, "" + }, + run: func(cc *collectContext) error { + return collectServiceLogs(cc, service) + }, + } +} + +// collectServiceLogs writes one log file per replica of a service into +// bundle/logs//, plus a .previous.log per restarted Kubernetes +// container. Per-replica failures never abort the remaining replicas; they +// are aggregated into the returned error so the manifest records the partial +// outcome while the successfully written logs stay in the bundle (FR-009). +func collectServiceLogs(cc *collectContext, service string) error { + cb, err := cc.collect() + if err != nil { + return err + } + + replicas, err := cb.ServiceReplicas(service) + if err != nil { + return fmt.Errorf("failed to enumerate %s replicas: %w", service, err) + } + if len(replicas) == 0 { + return fmt.Errorf("no replicas found for service %s", service) + } + + serviceDir := filepath.Join(cc.bundleDir, "logs", service) + if err := os.MkdirAll(serviceDir, 0755); err != nil { + return fmt.Errorf("failed to create log directory %s: %w", serviceDir, err) + } + + counts := podContainerCounts(replicas) + failures := []string{} + var timeout *timeoutError + for _, replica := range replicas { + multiContainer := counts[replica.Pod] > 1 + + filename := replicaLogFilename(replica, multiContainer, false) + if err := writeReplicaLog(cb, replica, cc.opts.LogLines, false, filepath.Join(serviceDir, filename)); err != nil { + logrus.Warnf("Failed to collect logs for %s: %v", filename, err) + failures = append(failures, fmt.Sprintf("%s: %v", filename, err)) + timeout = captureTimeout(timeout, err) + continue + } + + // Previous-container logs exist only for restarted Kubernetes + // containers; a failure fetching them keeps the current logs and is + // reported as a partial outcome. + if !replica.Restarted { + continue + } + previousName := replicaLogFilename(replica, multiContainer, true) + if err := writeReplicaLog(cb, replica, cc.opts.LogLines, true, filepath.Join(serviceDir, previousName)); err != nil { + logrus.Warnf("Previous logs unavailable for %s: %v", filename, err) + failures = append(failures, fmt.Sprintf("previous logs unavailable for %s: %v", filename, err)) + timeout = captureTimeout(timeout, err) + } + } + + return partialError(fmt.Sprintf("partial log collection for %s", service), failures, timeout) +} + +// podContainerCounts counts replicas per pod, detecting multi-container pods +// for filename derivation. Docker replicas carry no pod and never count. +func podContainerCounts(replicas []Replica) map[string]int { + counts := map[string]int{} + for _, replica := range replicas { + if replica.Pod == "" { + continue + } + counts[replica.Pod]++ + } + return counts +} + +// replicaBaseName derives the bundle-path base name for one replica per +// data-model.md: Docker → container name; Kubernetes single-container pod → +// pod name; multi-container pod → _. Log files and the +// per-replica task-worker directories share this derivation. +func replicaBaseName(replica Replica, multiContainer bool) string { + base := replica.Container + if replica.Pod != "" { + base = replica.Pod + if multiContainer { + base = replica.Pod + "_" + replica.Container + } + } + return base +} + +// replicaLogFilename derives the bundle log filename for one replica: +// .log, or .previous.log for previous-container logs. +func replicaLogFilename(replica Replica, multiContainer bool, previous bool) string { + base := replicaBaseName(replica, multiContainer) + if previous { + return base + ".previous.log" + } + return base + ".log" +} + +// writeReplicaLog streams one replica's logs to path, draining the reader +// before calling wait so large logs never buffer in memory. A failed fetch +// that produced no output removes the empty file so the bundle never carries +// misleading zero-byte logs. +func writeReplicaLog(cb collectBackend, replica Replica, tailLines int, previous bool, path string) error { + reader, wait, err := cb.ReplicaLogs(replica, tailLines, previous) + if err != nil { + return fmt.Errorf("failed to start log stream: %w", err) + } + // Always close the reader: on the Docker combined-pipe path it is the read + // end of an os.Pipe that leaks a file descriptor otherwise (FIX-6). On the + // kubectl stdout-pipe path Wait already closes it, so this is a harmless + // second close. + defer func() { + if closeErr := reader.Close(); closeErr != nil { + logrus.Debugf("Failed to close log stream for %s: %v", path, closeErr) + } + }() + + file, err := os.Create(path) + if err != nil { + if waitErr := wait(); waitErr != nil { + logrus.Debugf("Log command for %s exited with error: %v", path, waitErr) + } + return fmt.Errorf("failed to create log file %s: %w", path, err) + } + + written, copyErr := io.Copy(file, reader) + closeErr := file.Close() + waitErr := wait() + + if (copyErr != nil || waitErr != nil || closeErr != nil) && written == 0 { + if removeErr := os.Remove(path); removeErr != nil { + logrus.Warnf("Failed to remove empty log file %s: %v", path, removeErr) + } + } + + if copyErr != nil { + return fmt.Errorf("failed to stream logs to %s: %w", path, copyErr) + } + if waitErr != nil { + return fmt.Errorf("log command failed: %w", waitErr) + } + if closeErr != nil { + return fmt.Errorf("failed to write log file %s: %w", path, closeErr) + } + return nil +} diff --git a/src/internal/app/collect_logs_test.go b/src/internal/app/collect_logs_test.go new file mode 100644 index 0000000..a2b095f --- /dev/null +++ b/src/internal/app/collect_logs_test.go @@ -0,0 +1,427 @@ +package app + +import ( + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestReplicaLogFilename(t *testing.T) { + tests := []struct { + name string + replica Replica + multiContainer bool + previous bool + want string + }{ + { + name: "docker container", + replica: Replica{Service: "infrahub-server", Container: "infrahub-server-1"}, + want: "infrahub-server-1.log", + }, + { + name: "docker previous suffix replaces .log", + replica: Replica{Service: "cache", Container: "cache-1"}, + previous: true, + want: "cache-1.previous.log", + }, + { + name: "k8s single-container pod uses pod name", + replica: Replica{Service: "infrahub-server", Pod: "infrahub-server-0", Container: "infrahub-server"}, + want: "infrahub-server-0.log", + }, + { + name: "k8s multi-container pod appends container", + replica: Replica{Service: "task-manager-db", Pod: "task-manager-db-1", Container: "postgres"}, + multiContainer: true, + want: "task-manager-db-1_postgres.log", + }, + { + name: "k8s single-container previous", + replica: Replica{Service: "task-worker", Pod: "task-worker-0", Container: "task-worker"}, + previous: true, + want: "task-worker-0.previous.log", + }, + { + name: "k8s multi-container previous", + replica: Replica{Service: "task-manager-db", Pod: "task-manager-db-1", Container: "metrics-exporter"}, + multiContainer: true, + previous: true, + want: "task-manager-db-1_metrics-exporter.previous.log", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := replicaLogFilename(tt.replica, tt.multiContainer, tt.previous) + if got != tt.want { + t.Errorf("replicaLogFilename(%+v, %v, %v) = %q, want %q", tt.replica, tt.multiContainer, tt.previous, got, tt.want) + } + }) + } +} + +func TestPodContainerCounts(t *testing.T) { + replicas := []Replica{ + {Service: "infrahub-server", Pod: "infrahub-server-0", Container: "infrahub-server"}, + {Service: "task-manager-db", Pod: "task-manager-db-1", Container: "postgres"}, + {Service: "task-manager-db", Pod: "task-manager-db-1", Container: "metrics-exporter"}, + {Service: "cache", Container: "cache-1"}, // Docker replica: no pod + } + + counts := podContainerCounts(replicas) + + want := map[string]int{ + "infrahub-server-0": 1, + "task-manager-db-1": 2, + } + if !reflect.DeepEqual(counts, want) { + t.Errorf("podContainerCounts() = %v, want %v (Docker replicas must not count)", counts, want) + } +} + +func TestKubectlLogArgs(t *testing.T) { + replica := Replica{Service: "infrahub-server", Pod: "infrahub-server-0", Container: "infrahub-server"} + + tests := []struct { + name string + tailLines int + previous bool + want []string + }{ + { + name: "current logs", + tailLines: 100000, + want: []string{ + "logs", "-n", "infrahub", "infrahub-server-0", + "-c", "infrahub-server", "--tail=100000", + }, + }, + { + name: "previous logs append --previous", + tailLines: 500, + previous: true, + want: []string{ + "logs", "-n", "infrahub", "infrahub-server-0", + "-c", "infrahub-server", "--tail=500", "--previous", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := kubectlLogArgs("infrahub", replica, tt.tailLines, tt.previous) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("kubectlLogArgs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestKubectlContainerStatusArgs(t *testing.T) { + got := kubectlContainerStatusArgs("infrahub", "database-0") + want := []string{ + "get", "pod", "database-0", "-n", "infrahub", + "-o", "jsonpath={range .status.containerStatuses[*]}{.name}{\" \"}{.restartCount}{\"\\n\"}{end}", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("kubectlContainerStatusArgs() = %v, want %v", got, want) + } +} + +func TestParsePodContainerStatuses(t *testing.T) { + tests := []struct { + name string + output string + want []Replica + wantErr string + }{ + { + name: "single container without restarts", + output: "infrahub-server 0\n", + want: []Replica{ + {Service: "infrahub-server", Pod: "pod-a", Container: "infrahub-server", Restarted: false}, + }, + }, + { + name: "multi-container pod with per-container restart counts", + output: "postgres 0\nmetrics-exporter 3\n", + want: []Replica{ + {Service: "infrahub-server", Pod: "pod-a", Container: "postgres", Restarted: false}, + {Service: "infrahub-server", Pod: "pod-a", Container: "metrics-exporter", Restarted: true}, + }, + }, + { + name: "empty output yields no replicas", + output: "\n\n", + want: []Replica{}, + }, + { + name: "malformed line", + output: "just-one-field\n", + wantErr: "unexpected container status line", + }, + { + name: "non-numeric restart count", + output: "infrahub-server many\n", + wantErr: "invalid restart count", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parsePodContainerStatuses("infrahub-server", "pod-a", tt.output) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("parsePodContainerStatuses(%q) succeeded, want error containing %q", tt.output, tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error = %q, want it to contain %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("parsePodContainerStatuses(%q) failed: %v", tt.output, err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parsePodContainerStatuses(%q) = %+v, want %+v", tt.output, got, tt.want) + } + }) + } +} + +// logsFakeBackend is a collect backend whose log streams distinguish current +// from previous logs and can fail on demand. +type logsFakeBackend struct { + bareBackend + replicas map[string][]Replica + replicasErr error + logs map[string]string // keyed by "/" + previousLogs map[string]string + previousErr error +} + +var _ collectBackend = (*logsFakeBackend)(nil) + +func (f *logsFakeBackend) ServiceReplicas(service string) ([]Replica, error) { + if f.replicasErr != nil { + return nil, f.replicasErr + } + return f.replicas[service], nil +} + +func (f *logsFakeBackend) ReplicaLogs(replica Replica, tailLines int, previous bool) (io.ReadCloser, func() error, error) { + key := replica.Pod + "/" + replica.Container + if previous { + if f.previousErr != nil { + // Mimic kubectl: the process starts, produces no output, and the + // failure surfaces through wait(). + return io.NopCloser(strings.NewReader("")), func() error { return f.previousErr }, nil + } + return io.NopCloser(strings.NewReader(f.previousLogs[key])), func() error { return nil }, nil + } + return io.NopCloser(strings.NewReader(f.logs[key])), func() error { return nil }, nil +} + +func (f *logsFakeBackend) Metrics() (string, error) { + return "", nil +} + +func newLogsFakeBackend() *logsFakeBackend { + return &logsFakeBackend{ + bareBackend: bareBackend{name: "kubernetes"}, + replicas: map[string][]Replica{ + "infrahub-server": { + {Service: "infrahub-server", Pod: "infrahub-server-0", Container: "infrahub-server"}, + {Service: "infrahub-server", Pod: "infrahub-server-1", Container: "infrahub-server"}, + }, + "task-manager-db": { + {Service: "task-manager-db", Pod: "task-manager-db-1", Container: "metrics-exporter", Restarted: true}, + {Service: "task-manager-db", Pod: "task-manager-db-1", Container: "postgres"}, + }, + }, + logs: map[string]string{ + "infrahub-server-0/infrahub-server": "server zero\n", + "infrahub-server-1/infrahub-server": "server one\n", + "task-manager-db-1/postgres": "postgres current\n", + "task-manager-db-1/metrics-exporter": "exporter current\n", + }, + previousLogs: map[string]string{ + "task-manager-db-1/metrics-exporter": "exporter before crash\n", + }, + } +} + +func newLogsCollectContext(t *testing.T, backend EnvironmentBackend) *collectContext { + t.Helper() + return &collectContext{ + backend: backend, + opts: CollectOptions{LogLines: 100}, + bundleDir: t.TempDir(), + } +} + +func TestServiceLogCollectors_CanonicalNames(t *testing.T) { + collectors := serviceLogCollectors() + + wantNames := []string{ + "logs/infrahub-server", + "logs/task-worker", + "logs/database", + "logs/message-queue", + "logs/cache", + "logs/task-manager", + "logs/task-manager-db", + "logs/task-manager-background-svc", + } + if len(collectors) != len(wantNames) { + t.Fatalf("serviceLogCollectors() returned %d collectors, want %d", len(collectors), len(wantNames)) + } + for i, want := range wantNames { + if collectors[i].name != want { + t.Errorf("collectors[%d].name = %q, want %q", i, collectors[i].name, want) + } + } +} + +func TestCollectServiceLogs_PerReplicaFiles(t *testing.T) { + cc := newLogsCollectContext(t, newLogsFakeBackend()) + + if err := collectServiceLogs(cc, "infrahub-server"); err != nil { + t.Fatalf("collectServiceLogs(infrahub-server) failed: %v", err) + } + + wantFiles := map[string]string{ + "infrahub-server-0.log": "server zero\n", + "infrahub-server-1.log": "server one\n", + } + for name, wantContent := range wantFiles { + data, err := os.ReadFile(filepath.Join(cc.bundleDir, "logs", "infrahub-server", name)) + if err != nil { + t.Errorf("expected log file %s missing: %v", name, err) + continue + } + if string(data) != wantContent { + t.Errorf("%s content = %q, want %q", name, data, wantContent) + } + } +} + +func TestCollectServiceLogs_MultiContainerAndPrevious(t *testing.T) { + cc := newLogsCollectContext(t, newLogsFakeBackend()) + + if err := collectServiceLogs(cc, "task-manager-db"); err != nil { + t.Fatalf("collectServiceLogs(task-manager-db) failed: %v", err) + } + + serviceDir := filepath.Join(cc.bundleDir, "logs", "task-manager-db") + wantFiles := map[string]string{ + "task-manager-db-1_postgres.log": "postgres current\n", + "task-manager-db-1_metrics-exporter.log": "exporter current\n", + "task-manager-db-1_metrics-exporter.previous.log": "exporter before crash\n", + } + for name, wantContent := range wantFiles { + data, err := os.ReadFile(filepath.Join(serviceDir, name)) + if err != nil { + t.Errorf("expected log file %s missing: %v", name, err) + continue + } + if string(data) != wantContent { + t.Errorf("%s content = %q, want %q", name, data, wantContent) + } + } + + // The non-restarted container must not get a previous log file. + if _, err := os.Stat(filepath.Join(serviceDir, "task-manager-db-1_postgres.previous.log")); !os.IsNotExist(err) { + t.Errorf("previous log written for a container that never restarted (err = %v)", err) + } +} + +func TestCollectServiceLogs_PreviousLogFailureIsPartial(t *testing.T) { + backend := newLogsFakeBackend() + backend.previousErr = fmt.Errorf("exit status 1: previous terminated container not found") + cc := newLogsCollectContext(t, backend) + + err := collectServiceLogs(cc, "task-manager-db") + if err == nil { + t.Fatal("collectServiceLogs succeeded, want partial-failure error for previous logs") + } + if !strings.Contains(err.Error(), "previous logs unavailable") { + t.Errorf("error = %q, want mention of unavailable previous logs", err) + } + + // Current logs of every replica are still in the bundle. + serviceDir := filepath.Join(cc.bundleDir, "logs", "task-manager-db") + for _, name := range []string{"task-manager-db-1_postgres.log", "task-manager-db-1_metrics-exporter.log"} { + if _, statErr := os.Stat(filepath.Join(serviceDir, name)); statErr != nil { + t.Errorf("current log %s missing after previous-log failure: %v", name, statErr) + } + } + + // The failed previous fetch produced no output, so no empty file remains. + if _, statErr := os.Stat(filepath.Join(serviceDir, "task-manager-db-1_metrics-exporter.previous.log")); !os.IsNotExist(statErr) { + t.Errorf("empty previous log file left behind (err = %v)", statErr) + } +} + +func TestServiceLogCollector_SkipAndFailure(t *testing.T) { + t.Run("absent service is skipped with a reason", func(t *testing.T) { + cc := newLogsCollectContext(t, newLogsFakeBackend()) + c := serviceLogCollector("task-manager-background-svc") + + skip, reason := c.skip(cc) + if !skip { + t.Fatal("skip = false for a service with no replicas, want true") + } + if reason != "service not deployed" { + t.Errorf("skip reason = %q, want %q", reason, "service not deployed") + } + }) + + t.Run("deployed service is not skipped", func(t *testing.T) { + cc := newLogsCollectContext(t, newLogsFakeBackend()) + c := serviceLogCollector("infrahub-server") + + if skip, reason := c.skip(cc); skip { + t.Errorf("skip = true (%q) for a deployed service, want false", reason) + } + }) + + t.Run("enumeration error surfaces as run failure, not skip", func(t *testing.T) { + backend := newLogsFakeBackend() + backend.replicasErr = fmt.Errorf("connection to the server refused") + cc := newLogsCollectContext(t, backend) + c := serviceLogCollector("infrahub-server") + + if skip, _ := c.skip(cc); skip { + t.Fatal("skip = true on enumeration error, want false so run reports failed") + } + err := c.run(cc) + if err == nil { + t.Fatal("run succeeded despite enumeration error, want failure") + } + if !strings.Contains(err.Error(), "failed to enumerate infrahub-server replicas") { + t.Errorf("error = %q, want enumeration context", err) + } + }) + + t.Run("backend without collect primitives is not skipped and fails in run", func(t *testing.T) { + cc := newLogsCollectContext(t, &bareBackend{name: "docker"}) + c := serviceLogCollector("infrahub-server") + + if skip, _ := c.skip(cc); skip { + t.Fatal("skip = true for unsupported backend, want false so run reports failed") + } + err := c.run(cc) + if err == nil { + t.Fatal("run succeeded on a backend without collect primitives, want failure") + } + if !strings.Contains(err.Error(), "does not support bundle collection") { + t.Errorf("error = %q, want unsupported-backend message", err) + } + }) +} diff --git a/src/internal/app/collect_manifest.go b/src/internal/app/collect_manifest.go new file mode 100644 index 0000000..9434c83 --- /dev/null +++ b/src/internal/app/collect_manifest.go @@ -0,0 +1,111 @@ +package app + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" +) + +// manifestVersion is the date-serial version of the bundle manifest schema +// (bump on breaking manifest changes), mirroring metadataVersion for backups. +const manifestVersion = 2026070200 + +// bundleManifestFilename is the manifest file written at the bundle root. +const bundleManifestFilename = "bundle_information.json" + +// collectIDLayout formats collect identifiers as YYYYMMDD_HHMMSS (UTC). +const collectIDLayout = "20060102_150405" + +// collectorStatus is the outcome of a single collector run. +type collectorStatus string + +const ( + collectorStatusSuccess collectorStatus = "success" + collectorStatusFailed collectorStatus = "failed" + collectorStatusSkipped collectorStatus = "skipped" +) + +// CollectorResult records the outcome of one collector in the bundle +// manifest. Reason is required whenever Status is not success. Artifact +// optionally references a filesystem artifact the collector produced outside +// the bundle (e.g. the --include-backup archive path); it is only ever set +// on success. +type CollectorResult struct { + Name string `json:"name"` + Status collectorStatus `json:"status"` + Reason string `json:"reason,omitempty"` + Artifact string `json:"artifact,omitempty"` +} + +// BundleManifest is the collect-side sibling of BackupMetadata, serialized as +// bundle_information.json at the bundle root. JSON field names are normative +// (specs/003-collect-tool/contracts/manifest.schema.json). +type BundleManifest struct { + ManifestVersion int `json:"manifest_version"` + CollectID string `json:"collect_id"` + CreatedAt string `json:"created_at"` + ToolVersion string `json:"tool_version"` + InfrahubVersion string `json:"infrahub_version"` + Environment string `json:"environment"` + LogLines int `json:"log_lines"` + Collectors []CollectorResult `json:"collectors"` +} + +// generateCollectID returns a UTC timestamp identifier shared by the bundle +// manifest and the archive filename. +func generateCollectID() string { + return time.Now().UTC().Format(collectIDLayout) +} + +// newBundleManifest builds a manifest for a collect run. InfrahubVersion is +// populated later by the server-info collection step; an empty string is +// valid when the server is unreachable. +func newBundleManifest(collectID, environment string, logLines int) *BundleManifest { + return &BundleManifest{ + ManifestVersion: manifestVersion, + CollectID: collectID, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + ToolVersion: BuildRevision(), + Environment: environment, + LogLines: logLines, + Collectors: []CollectorResult{}, + } +} + +// recordSuccess appends a success outcome for a collector. +func (m *BundleManifest) recordSuccess(name string) { + m.Collectors = append(m.Collectors, CollectorResult{Name: name, Status: collectorStatusSuccess}) +} + +// recordSuccessArtifact appends a success outcome referencing a filesystem +// artifact produced outside the bundle (e.g. the --include-backup archive). +func (m *BundleManifest) recordSuccessArtifact(name, artifact string) { + m.Collectors = append(m.Collectors, CollectorResult{Name: name, Status: collectorStatusSuccess, Artifact: artifact}) +} + +// recordFailed appends a failed outcome with a human-readable reason. +func (m *BundleManifest) recordFailed(name, reason string) { + m.Collectors = append(m.Collectors, CollectorResult{Name: name, Status: collectorStatusFailed, Reason: reason}) +} + +// recordSkipped appends a skipped outcome with a human-readable reason. +func (m *BundleManifest) recordSkipped(name, reason string) { + m.Collectors = append(m.Collectors, CollectorResult{Name: name, Status: collectorStatusSkipped, Reason: reason}) +} + +// write serializes the manifest as bundle_information.json in bundleDir. +func (m *BundleManifest) write(bundleDir string) error { + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal bundle manifest: %w", err) + } + + path := filepath.Join(bundleDir, bundleManifestFilename) + if err := os.WriteFile(path, data, 0644); err != nil { + return fmt.Errorf("failed to write bundle manifest %s: %w", path, err) + } + + return nil +} diff --git a/src/internal/app/collect_manifest_test.go b/src/internal/app/collect_manifest_test.go new file mode 100644 index 0000000..2778d4b --- /dev/null +++ b/src/internal/app/collect_manifest_test.go @@ -0,0 +1,201 @@ +package app + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + "time" +) + +var collectIDPattern = regexp.MustCompile(`^[0-9]{8}_[0-9]{6}$`) + +func TestGenerateCollectID_Pattern(t *testing.T) { + id := generateCollectID() + if !collectIDPattern.MatchString(id) { + t.Errorf("generateCollectID() = %q, want match for %s", id, collectIDPattern) + } +} + +func TestNewBundleManifest_Fields(t *testing.T) { + before := time.Now().UTC() + manifest := newBundleManifest("20260703_101530", "kubernetes", 100000) + + if manifest.ManifestVersion != 2026070200 { + t.Errorf("ManifestVersion = %d, want 2026070200", manifest.ManifestVersion) + } + if manifest.CollectID != "20260703_101530" { + t.Errorf("CollectID = %q, want %q", manifest.CollectID, "20260703_101530") + } + if manifest.Environment != "kubernetes" { + t.Errorf("Environment = %q, want %q", manifest.Environment, "kubernetes") + } + if manifest.LogLines != 100000 { + t.Errorf("LogLines = %d, want 100000", manifest.LogLines) + } + if manifest.ToolVersion == "" { + t.Error("ToolVersion is empty, want BuildRevision output") + } + if manifest.Collectors == nil { + t.Error("Collectors is nil, want empty slice so JSON serializes as []") + } + + createdAt, err := time.Parse(time.RFC3339, manifest.CreatedAt) + if err != nil { + t.Fatalf("CreatedAt %q is not RFC3339: %v", manifest.CreatedAt, err) + } + if createdAt.Before(before.Truncate(time.Second)) || createdAt.After(time.Now().UTC().Add(time.Second)) { + t.Errorf("CreatedAt %q is not a current UTC timestamp", manifest.CreatedAt) + } +} + +// TestBundleManifest_JSONShape validates the serialized manifest against the +// semantics of specs/003-collect-tool/contracts/manifest.schema.json: exact +// field names, collect_id pattern, environment/status enums, and reason +// present exactly when status is failed or skipped. +func TestBundleManifest_JSONShape(t *testing.T) { + manifest := newBundleManifest(generateCollectID(), "docker", 500) + manifest.recordSuccess("logs/infrahub-server") + manifest.recordFailed("cache-status", "container not running") + manifest.recordSkipped("benchmark", "not requested") + manifest.recordFailed("message-queue-status", "timed out after 60s") + + data, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + t.Fatalf("MarshalIndent failed: %v", err) + } + + var decoded map[string]any + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + // All schema-required fields must be present with the exact JSON names. + // The schema allows additionalProperties, so the exact field count is not + // asserted — that would break the next time a legal optional field is added + // (FIX-T1). + for _, field := range []string{ + "manifest_version", "collect_id", "created_at", "tool_version", + "infrahub_version", "environment", "log_lines", "collectors", + } { + if _, ok := decoded[field]; !ok { + t.Errorf("serialized manifest is missing required field %q", field) + } + } + + if version, ok := decoded["manifest_version"].(float64); !ok || int(version) != 2026070200 { + t.Errorf("manifest_version = %v, want 2026070200", decoded["manifest_version"]) + } + if id, ok := decoded["collect_id"].(string); !ok || !collectIDPattern.MatchString(id) { + t.Errorf("collect_id = %v, want match for %s", decoded["collect_id"], collectIDPattern) + } + if env, ok := decoded["environment"].(string); !ok || (env != "docker" && env != "kubernetes") { + t.Errorf("environment = %v, want docker or kubernetes", decoded["environment"]) + } + if lines, ok := decoded["log_lines"].(float64); !ok || int(lines) != 500 { + t.Errorf("log_lines = %v, want 500", decoded["log_lines"]) + } + if infrahubVersion, ok := decoded["infrahub_version"].(string); !ok { + t.Errorf("infrahub_version = %v, want a string (empty allowed)", decoded["infrahub_version"]) + } else if infrahubVersion != "" { + t.Errorf("infrahub_version = %q, want empty before server collection populates it", infrahubVersion) + } + + collectors, ok := decoded["collectors"].([]any) + if !ok { + t.Fatalf("collectors = %T, want array", decoded["collectors"]) + } + if len(collectors) != 4 { + t.Fatalf("collectors has %d entries, want 4", len(collectors)) + } + + wantEntries := []struct { + name string + status string + wantReason string + hasReason bool + }{ + {"logs/infrahub-server", "success", "", false}, + {"cache-status", "failed", "container not running", true}, + {"benchmark", "skipped", "not requested", true}, + {"message-queue-status", "failed", "timed out after 60s", true}, + } + + validStatuses := map[string]bool{"success": true, "failed": true, "skipped": true} + + for i, want := range wantEntries { + entry, ok := collectors[i].(map[string]any) + if !ok { + t.Fatalf("collectors[%d] = %T, want object", i, collectors[i]) + } + if entry["name"] != want.name { + t.Errorf("collectors[%d].name = %v, want %q", i, entry["name"], want.name) + } + status, _ := entry["status"].(string) + if !validStatuses[status] { + t.Errorf("collectors[%d].status = %q, not in schema enum success|failed|skipped", i, status) + } + if status != want.status { + t.Errorf("collectors[%d].status = %q, want %q", i, status, want.status) + } + reason, hasReason := entry["reason"] + if hasReason != want.hasReason { + t.Errorf("collectors[%d] reason present = %v, want %v (schema: reason required iff status != success)", i, hasReason, want.hasReason) + } + if want.hasReason && reason != want.wantReason { + t.Errorf("collectors[%d].reason = %v, want %q", i, reason, want.wantReason) + } + } +} + +func TestBundleManifest_EmptyCollectorsSerializesAsArray(t *testing.T) { + manifest := newBundleManifest(generateCollectID(), "docker", 100000) + + data, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + var decoded map[string]any + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + if _, ok := decoded["collectors"].([]any); !ok { + t.Errorf("collectors = %v (%T), want JSON array (not null)", decoded["collectors"], decoded["collectors"]) + } +} + +func TestBundleManifest_Write(t *testing.T) { + bundleDir := t.TempDir() + manifest := newBundleManifest("20260703_101530", "docker", 100000) + manifest.recordSuccess("metrics") + + if err := manifest.write(bundleDir); err != nil { + t.Fatalf("write failed: %v", err) + } + + data, err := os.ReadFile(filepath.Join(bundleDir, "bundle_information.json")) + if err != nil { + t.Fatalf("bundle_information.json not written: %v", err) + } + + var decoded BundleManifest + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("written manifest is not valid JSON: %v", err) + } + if decoded.CollectID != "20260703_101530" { + t.Errorf("round-tripped CollectID = %q, want %q", decoded.CollectID, "20260703_101530") + } + if len(decoded.Collectors) != 1 || decoded.Collectors[0].Status != collectorStatusSuccess { + t.Errorf("round-tripped Collectors = %+v, want one success entry", decoded.Collectors) + } +} + +func TestBundleManifest_WriteFailsOnMissingDir(t *testing.T) { + manifest := newBundleManifest(generateCollectID(), "docker", 100000) + err := manifest.write(filepath.Join(t.TempDir(), "does", "not", "exist")) + if err == nil { + t.Fatal("write to missing directory succeeded, want error") + } +} diff --git a/src/internal/app/collect_metrics.go b/src/internal/app/collect_metrics.go new file mode 100644 index 0000000..ef6d929 --- /dev/null +++ b/src/internal/app/collect_metrics.go @@ -0,0 +1,42 @@ +package app + +import ( + "fmt" + "os" + "path/filepath" +) + +// metricsBundleDir is the bundle directory for one-shot resource metrics +// (docker stats / kubectl top; research R4). +const metricsBundleDir = "metrics" + +// metricsFilename is stable across environments so the bundle layout stays +// identical on Docker and Kubernetes (SC-004). +const metricsFilename = "metrics.txt" + +// metricsCollector captures one-shot container resource metrics through the +// backend Metrics primitive into bundle/metrics/. A missing metrics source +// (e.g. no metrics-server on Kubernetes) surfaces as a failed manifest entry, +// never a run failure (research R4). +func metricsCollector() collector { + return collector{ + name: "metrics", + run: func(cc *collectContext) error { + cb, err := cc.collect() + if err != nil { + return err + } + + output, err := cb.Metrics() + if err != nil { + return fmt.Errorf("failed to capture resource metrics: %w", err) + } + + dir := filepath.Join(cc.bundleDir, metricsBundleDir) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create %s directory: %w", metricsBundleDir, err) + } + return writeDumpFile(filepath.Join(dir, metricsFilename), output, nil) + }, + } +} diff --git a/src/internal/app/collect_test.go b/src/internal/app/collect_test.go new file mode 100644 index 0000000..6849611 --- /dev/null +++ b/src/internal/app/collect_test.go @@ -0,0 +1,624 @@ +package app + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// bareBackend implements EnvironmentBackend with inert stubs. It deliberately +// does NOT implement the collect primitives, standing in for a backend that +// has not been extended yet. +type bareBackend struct{ name string } + +func (b *bareBackend) Name() string { return b.name } +func (b *bareBackend) Detect() error { return nil } +func (b *bareBackend) Info() string { return "fake environment" } +func (b *bareBackend) Exec(service string, command []string, opts *ExecOptions) (string, error) { + return "", nil +} +func (b *bareBackend) ExecStream(service string, command []string, opts *ExecOptions) (string, error) { + return "", nil +} +func (b *bareBackend) ExecStreamPipe(service string, command []string, opts *ExecOptions) (io.ReadCloser, func() error, error) { + return io.NopCloser(strings.NewReader("")), func() error { return nil }, nil +} +func (b *bareBackend) ExecWritePipe(service string, command []string, opts *ExecOptions, stdin io.Reader) (func() error, error) { + return func() error { return nil }, nil +} +func (b *bareBackend) CopyTo(service, src, dest string) error { return nil } +func (b *bareBackend) CopyFrom(service, src, dest string) error { return nil } +func (b *bareBackend) Start(services ...string) error { return nil } +func (b *bareBackend) Stop(services ...string) error { return nil } +func (b *bareBackend) IsRunning(service string) (bool, error) { return true, nil } + +// fakeCollectBackend satisfies the full collectBackend seam for unit tests. +// execFn, when set, overrides Exec so tests can script per-command outputs. +type fakeCollectBackend struct { + bareBackend + replicas map[string][]Replica + logs map[string]string + metrics string + execFn func(service string, command []string, opts *ExecOptions) (string, error) +} + +func (f *fakeCollectBackend) Exec(service string, command []string, opts *ExecOptions) (string, error) { + if f.execFn != nil { + return f.execFn(service, command, opts) + } + return f.bareBackend.Exec(service, command, opts) +} + +func (f *fakeCollectBackend) ServiceReplicas(service string) ([]Replica, error) { + return f.replicas[service], nil +} + +func (f *fakeCollectBackend) ReplicaLogs(replica Replica, tailLines int, previous bool) (io.ReadCloser, func() error, error) { + return io.NopCloser(strings.NewReader(f.logs[replica.Container])), func() error { return nil }, nil +} + +func (f *fakeCollectBackend) Metrics() (string, error) { + return f.metrics, nil +} + +var ( + _ EnvironmentBackend = (*bareBackend)(nil) + _ collectBackend = (*fakeCollectBackend)(nil) +) + +func newFakeCollectBackend() *fakeCollectBackend { + return &fakeCollectBackend{ + bareBackend: bareBackend{name: "docker"}, + replicas: map[string][]Replica{ + "infrahub-server": { + {Service: "infrahub-server", Container: "infrahub-server-1"}, + }, + }, + logs: map[string]string{"infrahub-server-1": "log line\n"}, + metrics: "CONTAINER CPU% MEM%\ninfrahub-server-1 1% 2%\n", + } +} + +// extractBundleManifest extracts the archive and parses bundle/bundle_information.json. +func extractBundleManifest(t *testing.T, archivePath string) (*BundleManifest, string) { + t.Helper() + + destDir := t.TempDir() + if err := extractTarball(archivePath, destDir); err != nil { + t.Fatalf("failed to extract archive %s: %v", archivePath, err) + } + + data, err := os.ReadFile(filepath.Join(destDir, "bundle", "bundle_information.json")) + if err != nil { + t.Fatalf("bundle_information.json missing from archive: %v", err) + } + + var manifest BundleManifest + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatalf("manifest is not valid JSON: %v", err) + } + return &manifest, destDir +} + +// assertStagingRemoved verifies no infrahub_collect_* staging dir remains. +func assertStagingRemoved(t *testing.T, outputDir string) { + t.Helper() + leftovers, err := filepath.Glob(filepath.Join(outputDir, "infrahub_collect_*")) + if err != nil { + t.Fatalf("glob failed: %v", err) + } + if len(leftovers) != 0 { + t.Errorf("staging directories left behind: %v", leftovers) + } +} + +func findArchive(t *testing.T, outputDir string) string { + t.Helper() + archives, err := filepath.Glob(filepath.Join(outputDir, "support_bundle_*.tar.gz")) + if err != nil { + t.Fatalf("glob failed: %v", err) + } + if len(archives) != 1 { + t.Fatalf("found %d archives in %s, want exactly 1: %v", len(archives), outputDir, archives) + } + return archives[0] +} + +func TestRunCollectPlan_OutcomeRecording(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 500} + + epsilonRan := false + plan := []collector{ + { + name: "metrics", + run: func(cc *collectContext) error { + cb, err := cc.collect() + if err != nil { + return err + } + metrics, err := cb.Metrics() + if err != nil { + return err + } + return os.WriteFile(filepath.Join(cc.bundleDir, "metrics.txt"), []byte(metrics), 0644) + }, + }, + { + name: "cache-status", + run: func(cc *collectContext) error { + return fmt.Errorf("cache status dump: %w", fmt.Errorf("container not running")) + }, + }, + { + name: "benchmark", + skip: func(cc *collectContext) (bool, string) { + return !cc.opts.Benchmark, "not requested" + }, + run: func(cc *collectContext) error { return nil }, + }, + { + name: "message-queue-status", + run: func(cc *collectContext) error { + _, err := cc.iops.executor.runCommandContext(cc.ctx, 100*time.Millisecond, "sleep", "5") + if err != nil { + return fmt.Errorf("rabbitmqctl status: %w", err) + } + return nil + }, + }, + { + name: "logs/infrahub-server", + run: func(cc *collectContext) error { + epsilonRan = true + cb, err := cc.collect() + if err != nil { + return err + } + replicas, err := cb.ServiceReplicas("infrahub-server") + if err != nil { + return err + } + for _, replica := range replicas { + reader, wait, err := cb.ReplicaLogs(replica, cc.opts.LogLines, false) + if err != nil { + return err + } + content, err := io.ReadAll(reader) + if err != nil { + return err + } + if err := wait(); err != nil { + return err + } + logPath := filepath.Join(cc.bundleDir, replica.Container+".log") + if err := os.WriteFile(logPath, content, 0644); err != nil { + return err + } + } + return nil + }, + }, + } + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + + if !epsilonRan { + t.Error("collector after failures did not run; run must continue past failures (FR-009)") + } + + archivePath := findArchive(t, outputDir) + manifest, destDir := extractBundleManifest(t, archivePath) + + // The manifest accounts for every planned collector, in order (SC-005). + if len(manifest.Collectors) != len(plan) { + t.Fatalf("manifest has %d collector entries, want %d (one per planned collector)", len(manifest.Collectors), len(plan)) + } + + wantResults := []CollectorResult{ + {Name: "metrics", Status: collectorStatusSuccess}, + {Name: "cache-status", Status: collectorStatusFailed, Reason: "cache status dump: container not running"}, + {Name: "benchmark", Status: collectorStatusSkipped, Reason: "not requested"}, + {Name: "message-queue-status", Status: collectorStatusFailed, Reason: "timed out after 100ms"}, + {Name: "logs/infrahub-server", Status: collectorStatusSuccess}, + } + for i, want := range wantResults { + if manifest.Collectors[i] != want { + t.Errorf("collectors[%d] = %+v, want %+v", i, manifest.Collectors[i], want) + } + } + + if manifest.Environment != "docker" { + t.Errorf("manifest environment = %q, want %q", manifest.Environment, "docker") + } + if manifest.LogLines != 500 { + t.Errorf("manifest log_lines = %d, want 500", manifest.LogLines) + } + if !strings.Contains(archivePath, manifest.CollectID) { + t.Errorf("archive filename %q does not embed collect_id %q", archivePath, manifest.CollectID) + } + + // Files written by successful collectors are inside the archive under bundle/. + metricsData, err := os.ReadFile(filepath.Join(destDir, "bundle", "metrics.txt")) + if err != nil { + t.Errorf("metrics.txt missing from archive: %v", err) + } else if string(metricsData) != backend.metrics { + t.Errorf("metrics.txt content = %q, want %q", metricsData, backend.metrics) + } + logData, err := os.ReadFile(filepath.Join(destDir, "bundle", "infrahub-server-1.log")) + if err != nil { + t.Errorf("infrahub-server-1.log missing from archive: %v", err) + } else if string(logData) != "log line\n" { + t.Errorf("log content = %q, want %q", logData, "log line\n") + } + + assertStagingRemoved(t, outputDir) +} + +// TestCollectBundle_RegisteredPlan runs the full registered plan (T025) +// against the fake backend: only infrahub-server has replicas, so its logs, +// server info, and metrics are collected while every other service-bound +// collector is skipped as not deployed. +func TestCollectBundle_RegisteredPlan(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + backend.execFn = func(service string, command []string, opts *ExecOptions) (string, error) { + joined := strings.Join(command, " ") + switch { + case strings.Contains(joined, "infrahub.__version__"): + return "1.5.2", nil + case strings.Contains(joined, "INFRAHUB_INTERNAL_ADDRESS"): + return "http://infrahub-server:8000", nil + case joined == "pip list": + return "infrahub 1.5.2", nil + case joined == "env": + return "INFRAHUB_API_TOKEN=secret123\nINFRAHUB_HOST=localhost", nil + case strings.HasSuffix(joined, "/api/config"): + return `{"security":{"secret_key":"abc123"}}`, nil + case strings.HasSuffix(joined, "/api/info"): + return `{"version":"1.5.2"}`, nil + case strings.HasSuffix(joined, "/api/schema"): + return `{"nodes":[]}`, nil + default: + return "", nil + } + } + iops.backend = backend + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + if err := iops.CollectBundle(opts); err != nil { + t.Fatalf("CollectBundle failed: %v", err) + } + + archivePath := findArchive(t, outputDir) + manifest, destDir := extractBundleManifest(t, archivePath) + + if manifest.ManifestVersion != manifestVersion { + t.Errorf("manifest_version = %d, want %d", manifest.ManifestVersion, manifestVersion) + } + if manifest.Environment != "docker" { + t.Errorf("environment = %q, want %q", manifest.Environment, "docker") + } + if manifest.InfrahubVersion != "1.5.2" { + t.Errorf("infrahub_version = %q, want %q (populated by server-info)", manifest.InfrahubVersion, "1.5.2") + } + if manifest.LogLines != 100000 { + t.Errorf("log_lines = %d, want 100000", manifest.LogLines) + } + + notDeployed := "service not deployed" + wantResults := []CollectorResult{ + {Name: "logs/infrahub-server", Status: collectorStatusSuccess}, + {Name: "logs/task-worker", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "logs/database", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "logs/message-queue", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "logs/cache", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "logs/task-manager", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "logs/task-manager-db", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "logs/task-manager-background-svc", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "database-logs", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "message-queue-status", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "cache-status", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "task-worker-state", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "task-manager-state", Status: collectorStatusSkipped, Reason: notDeployed}, + {Name: "server-info", Status: collectorStatusSuccess}, + {Name: "metrics", Status: collectorStatusSuccess}, + {Name: "benchmark", Status: collectorStatusSkipped, Reason: "not requested"}, + {Name: "backup", Status: collectorStatusSkipped, Reason: "not requested"}, + } + if len(manifest.Collectors) != len(wantResults) { + t.Fatalf("manifest has %d collector entries %+v, want %d", len(manifest.Collectors), manifest.Collectors, len(wantResults)) + } + for i, want := range wantResults { + if manifest.Collectors[i] != want { + t.Errorf("collectors[%d] = %+v, want %+v", i, manifest.Collectors[i], want) + } + } + + // Server dumps are masked before they reach the bundle (research R5). + envDump, err := os.ReadFile(filepath.Join(destDir, "bundle", "server", "environment.txt")) + if err != nil { + t.Errorf("server/environment.txt missing from archive: %v", err) + } else { + if strings.Contains(string(envDump), "secret123") { + t.Errorf("environment.txt leaks the API token: %q", envDump) + } + if !strings.Contains(string(envDump), "INFRAHUB_API_TOKEN="+maskedValue) { + t.Errorf("environment.txt = %q, want the token masked", envDump) + } + } + configDump, err := os.ReadFile(filepath.Join(destDir, "bundle", "server", "config.json")) + if err != nil { + t.Errorf("server/config.json missing from archive: %v", err) + } else if strings.Contains(string(configDump), "abc123") || !strings.Contains(string(configDump), maskedValue) { + t.Errorf("config.json = %q, want the secret masked", configDump) + } + versionDump, err := os.ReadFile(filepath.Join(destDir, "bundle", "server", "version.txt")) + if err != nil { + t.Errorf("server/version.txt missing from archive: %v", err) + } else if strings.TrimSpace(string(versionDump)) != "1.5.2" { + t.Errorf("version.txt = %q, want %q", versionDump, "1.5.2") + } + + logDump, err := os.ReadFile(filepath.Join(destDir, "bundle", "logs", "infrahub-server", "infrahub-server-1.log")) + if err != nil { + t.Errorf("logs/infrahub-server/infrahub-server-1.log missing from archive: %v", err) + } else if string(logDump) != "log line\n" { + t.Errorf("log content = %q, want %q", logDump, "log line\n") + } + + metricsDump, err := os.ReadFile(filepath.Join(destDir, "bundle", "metrics", "metrics.txt")) + if err != nil { + t.Errorf("metrics/metrics.txt missing from archive: %v", err) + } else if !strings.HasPrefix(string(metricsDump), backend.metrics) { + t.Errorf("metrics.txt content = %q, want prefix %q", metricsDump, backend.metrics) + } + + assertStagingRemoved(t, outputDir) +} + +func TestRunCollectPlan_UnwritableOutputDir(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + + blocker := filepath.Join(t.TempDir(), "blocker") + if err := os.WriteFile(blocker, []byte("file, not dir"), 0644); err != nil { + t.Fatal(err) + } + + opts := CollectOptions{OutputDir: filepath.Join(blocker, "bundles"), LogLines: 100000} + err := iops.runCollectPlan(backend, opts, nil) + if err == nil { + t.Fatal("runCollectPlan with unwritable output dir succeeded, want hard error") + } + if !strings.Contains(err.Error(), "output directory") { + t.Errorf("error %q does not name the output directory", err) + } +} + +func TestRunCollectPlan_StagingRemovedOnFailure(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + // Sabotage the staging bundle dir so manifest finalization fails, + // exercising the failure-path cleanup. + plan := []collector{ + { + name: "saboteur", + run: func(cc *collectContext) error { + return os.RemoveAll(cc.bundleDir) + }, + }, + } + + err := iops.runCollectPlan(backend, opts, plan) + if err == nil { + t.Fatal("runCollectPlan succeeded, want manifest write failure") + } + + assertStagingRemoved(t, outputDir) + + archives, globErr := filepath.Glob(filepath.Join(outputDir, "support_bundle_*.tar.gz")) + if globErr != nil { + t.Fatal(globErr) + } + if len(archives) != 0 { + t.Errorf("failed run left archives behind: %v", archives) + } +} + +func TestRunCollectPlan_InterruptCleansStaging(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + plan := []collector{ + { + name: "interrupter", + run: func(cc *collectContext) error { + process, err := os.FindProcess(os.Getpid()) + if err != nil { + return err + } + if err := process.Signal(os.Interrupt); err != nil { + return err + } + select { + case <-cc.ctx.Done(): + return fmt.Errorf("stopped by interrupt") + case <-time.After(5 * time.Second): + return fmt.Errorf("interrupt signal was never delivered") + } + }, + }, + { + name: "never-reached", + run: func(cc *collectContext) error { + t.Error("collector ran after interrupt, want run aborted") + return nil + }, + }, + } + + err := iops.runCollectPlan(backend, opts, plan) + if err == nil { + t.Fatal("runCollectPlan succeeded, want interrupt error") + } + if !strings.Contains(err.Error(), "interrupted") { + t.Errorf("error = %q, want mention of interruption", err) + } + + assertStagingRemoved(t, outputDir) + + archives, globErr := filepath.Glob(filepath.Join(outputDir, "support_bundle_*.tar.gz")) + if globErr != nil { + t.Fatal(globErr) + } + if len(archives) != 0 { + t.Errorf("interrupted run left an archive with a final name: %v", archives) + } +} + +// TestRunCollectPlan_CollectorPanicRecovered proves a panicking collector (run +// or skip) is recorded as failed and the run continues, so one bad collector +// never defeats FR-009 (FIX-3). +func TestRunCollectPlan_CollectorPanicRecovered(t *testing.T) { + t.Run("a panic in run is recorded and the run continues", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + afterRan := false + plan := []collector{ + {name: "panicker", run: func(cc *collectContext) error { panic("boom") }}, + {name: "after", run: func(cc *collectContext) error { afterRan = true; return nil }}, + } + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan aborted on a collector panic, want the run to complete: %v", err) + } + if !afterRan { + t.Error("collector after the panicking one did not run (FR-009)") + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + wantResults := []CollectorResult{ + {Name: "panicker", Status: collectorStatusFailed, Reason: "panic: boom"}, + {Name: "after", Status: collectorStatusSuccess}, + } + if len(manifest.Collectors) != len(wantResults) { + t.Fatalf("manifest has %d entries %+v, want %d", len(manifest.Collectors), manifest.Collectors, len(wantResults)) + } + for i, want := range wantResults { + if manifest.Collectors[i] != want { + t.Errorf("collectors[%d] = %+v, want %+v", i, manifest.Collectors[i], want) + } + } + assertStagingRemoved(t, outputDir) + }) + + t.Run("a panic in the skip precondition is recorded as failed", func(t *testing.T) { + iops := NewInfrahubOps() + backend := newFakeCollectBackend() + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + plan := []collector{ + { + name: "panic-skip", + skip: func(cc *collectContext) (bool, string) { panic("skip boom") }, + run: func(cc *collectContext) error { t.Error("run reached despite panicking skip"); return nil }, + }, + } + + if err := iops.runCollectPlan(backend, opts, plan); err != nil { + t.Fatalf("runCollectPlan aborted on a panicking skip, want the run to complete: %v", err) + } + + manifest, _ := extractBundleManifest(t, findArchive(t, outputDir)) + want := CollectorResult{Name: "panic-skip", Status: collectorStatusFailed, Reason: "panic: skip boom"} + if len(manifest.Collectors) != 1 || manifest.Collectors[0] != want { + t.Errorf("collectors = %+v, want [%+v]", manifest.Collectors, want) + } + }) +} + +// readOnlyGuardBackend fails the test if any workload-lifecycle method is +// called, enforcing FR-010 / SC-003: collect must never stop, start, or scale +// a deployment's workloads (FIX-T3). +type readOnlyGuardBackend struct { + fakeCollectBackend + t *testing.T +} + +func (b *readOnlyGuardBackend) Start(services ...string) error { + b.t.Errorf("Start(%v) called during collect: FR-010 requires a read-only run", services) + return nil +} + +func (b *readOnlyGuardBackend) Stop(services ...string) error { + b.t.Errorf("Stop(%v) called during collect: FR-010 requires a read-only run", services) + return nil +} + +// TestRunCollectPlan_ReadOnlyGuard drives the full registered plan through +// runCollectPlan with a backend that fails the test on any lifecycle call, and +// asserts the bundle is still produced with zero Start/Stop calls (FR-010, +// SC-003 — previously only covered by the environment-gated k8s e2e; FIX-T3). +func TestRunCollectPlan_ReadOnlyGuard(t *testing.T) { + iops := NewInfrahubOps() + guard := &readOnlyGuardBackend{fakeCollectBackend: *newFakeCollectBackend(), t: t} + iops.backend = guard + outputDir := filepath.Join(t.TempDir(), "bundles") + opts := CollectOptions{OutputDir: outputDir, LogLines: 100000} + + if err := iops.runCollectPlan(guard, opts, iops.collectPlan(opts)); err != nil { + t.Fatalf("runCollectPlan failed: %v", err) + } + + // The guard asserts zero lifecycle calls; the archive must still exist. + findArchive(t, outputDir) + assertStagingRemoved(t, outputDir) +} + +func TestCollectContext_BackendSeam(t *testing.T) { + t.Run("bare backend is rejected with a clear error", func(t *testing.T) { + cc := &collectContext{backend: &bareBackend{name: "docker"}} + _, err := cc.collect() + if err == nil { + t.Fatal("collect() on a backend without collect primitives succeeded, want error") + } + if !strings.Contains(err.Error(), "does not support bundle collection") { + t.Errorf("error = %q, want a clear unsupported-backend message", err) + } + }) + + t.Run("collect-capable backend resolves", func(t *testing.T) { + cc := &collectContext{backend: newFakeCollectBackend()} + cb, err := cc.collect() + if err != nil { + t.Fatalf("collect() failed: %v", err) + } + replicas, err := cb.ServiceReplicas("infrahub-server") + if err != nil { + t.Fatalf("ServiceReplicas failed: %v", err) + } + if len(replicas) != 1 || replicas[0].Container != "infrahub-server-1" { + t.Errorf("replicas = %+v, want the fake's single infrahub-server replica", replicas) + } + }) +} diff --git a/src/internal/app/command_executor.go b/src/internal/app/command_executor.go index 4c8e2fc..fc0838b 100644 --- a/src/internal/app/command_executor.go +++ b/src/internal/app/command_executor.go @@ -2,11 +2,15 @@ package app import ( "bytes" + "context" + "errors" "fmt" "io" + "os" "os/exec" "strings" "sync" + "time" "github.com/sirupsen/logrus" ) @@ -60,6 +64,136 @@ func (ce *CommandExecutor) runCommand(name string, args ...string) (string, erro return strings.TrimSpace(string(output)), err } +// timeoutError marks a command that exceeded its allotted execution time. Its +// message is exactly "timed out after " so orchestrators can surface +// it verbatim (e.g. as a bundle manifest failure reason). +type timeoutError struct { + timeout time.Duration +} + +func (e *timeoutError) Error() string { + return "timed out after " + formatCommandTimeout(e.timeout) +} + +// formatCommandTimeout renders whole-second durations as plain seconds +// ("60s", "300s") to match the documented reason format; sub-second +// durations fall back to Go's default formatting. +func formatCommandTimeout(d time.Duration) string { + if d == d.Truncate(time.Second) { + return fmt.Sprintf("%ds", int64(d/time.Second)) + } + return d.String() +} + +// runCommandContext is the timeout-bounded variant of runCommand: the command +// is killed once timeout elapses (or ctx is cancelled) and the returned error +// is a *timeoutError when the timeout expired. +func (ce *CommandExecutor) runCommandContext(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + cctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(cctx, name, args...) + output, err := cmd.CombinedOutput() + if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { + return strings.TrimSpace(string(output)), &timeoutError{timeout: timeout} + } + return strings.TrimSpace(string(output)), err +} + +// runCommandPipeContext is the timeout-bounded variant of runCommandPipe. The +// caller must read from stdout and then call wait() to get the exit status; +// wait() returns a *timeoutError when the timeout expired before the command +// finished. The internal context is released when wait() is called. +func (ce *CommandExecutor) runCommandPipeContext(ctx context.Context, timeout time.Duration, name string, args ...string) (io.ReadCloser, func() error, error) { + cctx, cancel := context.WithTimeout(ctx, timeout) + + cmd := exec.CommandContext(cctx, name, args...) + logrus.Debugf("exec pipe (timeout %s): %s %s", timeout, name, strings.Join(args, " ")) + + stdout, err := cmd.StdoutPipe() + if err != nil { + cancel() + return nil, nil, err + } + + // Capture stderr for error reporting + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf + + if err := cmd.Start(); err != nil { + cancel() + return nil, nil, err + } + + wait := func() error { + defer cancel() + if err := cmd.Wait(); err != nil { + if errors.Is(cctx.Err(), context.DeadlineExceeded) { + return &timeoutError{timeout: timeout} + } + stderrStr := strings.TrimSpace(stderrBuf.String()) + if stderrStr != "" { + return fmt.Errorf("%w: %s", err, stderrStr) + } + return err + } + return nil + } + + return stdout, wait, nil +} + +// runCommandCombinedPipeContext is runCommandPipeContext with the command's +// stderr merged into the returned stream. `docker logs` demuxes a container's +// output onto the matching process streams and Infrahub services log to +// stderr, so capturing stdout alone would drop most of the log content. The +// caller must drain the reader and then call wait(); wait() returns a +// *timeoutError when the timeout expired before the command finished. +func (ce *CommandExecutor) runCommandCombinedPipeContext(ctx context.Context, timeout time.Duration, name string, args ...string) (io.ReadCloser, func() error, error) { + cctx, cancel := context.WithTimeout(ctx, timeout) + + cmd := exec.CommandContext(cctx, name, args...) + logrus.Debugf("exec combined pipe (timeout %s): %s %s", timeout, name, strings.Join(args, " ")) + + reader, writer, err := os.Pipe() + if err != nil { + cancel() + return nil, nil, err + } + cmd.Stdout = writer + cmd.Stderr = writer + + if err := cmd.Start(); err != nil { + cancel() + if closeErr := reader.Close(); closeErr != nil { + logrus.Debugf("Failed to close pipe reader: %v", closeErr) + } + if closeErr := writer.Close(); closeErr != nil { + logrus.Debugf("Failed to close pipe writer: %v", closeErr) + } + return nil, nil, err + } + + // The child process holds its own descriptor; closing the parent's copy + // lets the reader observe EOF as soon as the command exits. + if closeErr := writer.Close(); closeErr != nil { + logrus.Debugf("Failed to close pipe writer: %v", closeErr) + } + + wait := func() error { + defer cancel() + if err := cmd.Wait(); err != nil { + if errors.Is(cctx.Err(), context.DeadlineExceeded) { + return &timeoutError{timeout: timeout} + } + return err + } + return nil + } + + return reader, wait, nil +} + func (ce *CommandExecutor) runCommandQuiet(name string, args ...string) error { cmd := exec.Command(name, args...) return cmd.Run() diff --git a/src/internal/app/command_executor_test.go b/src/internal/app/command_executor_test.go new file mode 100644 index 0000000..a96c2e1 --- /dev/null +++ b/src/internal/app/command_executor_test.go @@ -0,0 +1,231 @@ +package app + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "time" +) + +func TestFormatCommandTimeout(t *testing.T) { + tests := []struct { + name string + d time.Duration + want string + }{ + {"exec dump default", collectExecTimeout, "60s"}, + {"transfer default", collectTransferTimeout, "300s"}, + {"sub-second", 100 * time.Millisecond, "100ms"}, + {"fractional seconds", 1500 * time.Millisecond, "1.5s"}, + {"one second", time.Second, "1s"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatCommandTimeout(tt.d); got != tt.want { + t.Errorf("formatCommandTimeout(%v) = %q, want %q", tt.d, got, tt.want) + } + }) + } +} + +func TestTimeoutError_Message(t *testing.T) { + err := &timeoutError{timeout: collectExecTimeout} + if err.Error() != "timed out after 60s" { + t.Errorf("Error() = %q, want %q", err.Error(), "timed out after 60s") + } +} + +func TestRunCommandContext_Success(t *testing.T) { + ce := NewCommandExecutor() + output, err := ce.runCommandContext(context.Background(), 10*time.Second, "echo", "hello") + if err != nil { + t.Fatalf("runCommandContext failed: %v", err) + } + if output != "hello" { + t.Errorf("output = %q, want %q", output, "hello") + } +} + +func TestRunCommandContext_Timeout(t *testing.T) { + ce := NewCommandExecutor() + start := time.Now() + _, err := ce.runCommandContext(context.Background(), 100*time.Millisecond, "sleep", "5") + elapsed := time.Since(start) + + if err == nil { + t.Fatal("runCommandContext did not fail, want timeout error") + } + var timeout *timeoutError + if !errors.As(err, &timeout) { + t.Fatalf("error = %v (%T), want *timeoutError", err, err) + } + if err.Error() != "timed out after 100ms" { + t.Errorf("Error() = %q, want %q", err.Error(), "timed out after 100ms") + } + if elapsed > 3*time.Second { + t.Errorf("command was not killed at the timeout: took %v", elapsed) + } +} + +func TestRunCommandContext_NonTimeoutFailure(t *testing.T) { + ce := NewCommandExecutor() + _, err := ce.runCommandContext(context.Background(), 10*time.Second, "sh", "-c", "exit 3") + if err == nil { + t.Fatal("runCommandContext succeeded, want exit error") + } + var timeout *timeoutError + if errors.As(err, &timeout) { + t.Errorf("plain command failure was reported as a timeout: %v", err) + } +} + +func TestRunCommandContext_ParentCancellationIsNotTimeout(t *testing.T) { + ce := NewCommandExecutor() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := ce.runCommandContext(ctx, 10*time.Second, "echo", "hello") + if err == nil { + t.Fatal("runCommandContext with cancelled parent succeeded, want error") + } + var timeout *timeoutError + if errors.As(err, &timeout) { + t.Errorf("parent cancellation was reported as a timeout: %v", err) + } +} + +func TestRunCommandPipeContext_Success(t *testing.T) { + ce := NewCommandExecutor() + stdout, wait, err := ce.runCommandPipeContext(context.Background(), 10*time.Second, "sh", "-c", "printf 'a\\nb\\n'") + if err != nil { + t.Fatalf("runCommandPipeContext failed to start: %v", err) + } + + content, err := io.ReadAll(stdout) + if err != nil { + t.Fatalf("reading stdout failed: %v", err) + } + if err := wait(); err != nil { + t.Fatalf("wait failed: %v", err) + } + if string(content) != "a\nb\n" { + t.Errorf("output = %q, want %q", content, "a\nb\n") + } +} + +func TestRunCommandPipeContext_Timeout(t *testing.T) { + ce := NewCommandExecutor() + stdout, wait, err := ce.runCommandPipeContext(context.Background(), 100*time.Millisecond, "sh", "-c", "echo started; exec sleep 5") + if err != nil { + t.Fatalf("runCommandPipeContext failed to start: %v", err) + } + + content, err := io.ReadAll(stdout) + if err != nil { + t.Fatalf("reading stdout failed: %v", err) + } + if !strings.Contains(string(content), "started") { + t.Errorf("output before timeout = %q, want to contain %q", content, "started") + } + + err = wait() + if err == nil { + t.Fatal("wait succeeded, want timeout error") + } + var timeout *timeoutError + if !errors.As(err, &timeout) { + t.Fatalf("error = %v (%T), want *timeoutError", err, err) + } + if err.Error() != "timed out after 100ms" { + t.Errorf("Error() = %q, want %q", err.Error(), "timed out after 100ms") + } +} + +func TestRunCommandCombinedPipeContext_MergesStderr(t *testing.T) { + ce := NewCommandExecutor() + reader, wait, err := ce.runCommandCombinedPipeContext(context.Background(), 10*time.Second, "sh", "-c", "printf 'out\\n'; printf 'err\\n' >&2") + if err != nil { + t.Fatalf("runCommandCombinedPipeContext failed to start: %v", err) + } + + content, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("reading combined output failed: %v", err) + } + if err := wait(); err != nil { + t.Fatalf("wait failed: %v", err) + } + for _, want := range []string{"out\n", "err\n"} { + if !strings.Contains(string(content), want) { + t.Errorf("combined output = %q, want it to contain %q", content, want) + } + } +} + +func TestRunCommandCombinedPipeContext_ReaderSeesEOFWhenCommandExits(t *testing.T) { + ce := NewCommandExecutor() + reader, wait, err := ce.runCommandCombinedPipeContext(context.Background(), 10*time.Second, "echo", "done") + if err != nil { + t.Fatalf("runCommandCombinedPipeContext failed to start: %v", err) + } + + // io.ReadAll only returns if the parent's write end was closed correctly; + // a leaked descriptor would block this read forever (test timeout). + if _, err := io.ReadAll(reader); err != nil { + t.Fatalf("reading combined output failed: %v", err) + } + if err := wait(); err != nil { + t.Fatalf("wait failed: %v", err) + } +} + +func TestRunCommandCombinedPipeContext_Timeout(t *testing.T) { + ce := NewCommandExecutor() + reader, wait, err := ce.runCommandCombinedPipeContext(context.Background(), 100*time.Millisecond, "sh", "-c", "echo started; exec sleep 5") + if err != nil { + t.Fatalf("runCommandCombinedPipeContext failed to start: %v", err) + } + + content, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("reading combined output failed: %v", err) + } + if !strings.Contains(string(content), "started") { + t.Errorf("output before timeout = %q, want to contain %q", content, "started") + } + + err = wait() + if err == nil { + t.Fatal("wait succeeded, want timeout error") + } + var timeout *timeoutError + if !errors.As(err, &timeout) { + t.Fatalf("error = %v (%T), want *timeoutError", err, err) + } +} + +func TestRunCommandPipeContext_CommandFailureKeepsStderr(t *testing.T) { + ce := NewCommandExecutor() + stdout, wait, err := ce.runCommandPipeContext(context.Background(), 10*time.Second, "sh", "-c", "echo oops >&2; exit 2") + if err != nil { + t.Fatalf("runCommandPipeContext failed to start: %v", err) + } + if _, err := io.ReadAll(stdout); err != nil { + t.Fatalf("reading stdout failed: %v", err) + } + + err = wait() + if err == nil { + t.Fatal("wait succeeded, want exit error") + } + var timeout *timeoutError + if errors.As(err, &timeout) { + t.Errorf("plain command failure was reported as a timeout: %v", err) + } + if !strings.Contains(err.Error(), "oops") { + t.Errorf("error = %q, want stderr content included", err) + } +} diff --git a/src/internal/app/environment_docker.go b/src/internal/app/environment_docker.go index e1e1368..0dec583 100644 --- a/src/internal/app/environment_docker.go +++ b/src/internal/app/environment_docker.go @@ -1,10 +1,16 @@ package app import ( + "context" + "encoding/json" "fmt" "io" "sort" + "strconv" "strings" + "time" + + "github.com/sirupsen/logrus" ) type DockerBackend struct { @@ -103,6 +109,254 @@ func (d *DockerBackend) ExecStream(service string, command []string, opts *ExecO return d.executor.runCommandWithStream("docker", d.buildExecArgs(service, command, opts)...) } +// DockerBackend implements the collect-side primitives (spec 003-collect-tool, +// research R3/R4) and the optional timeout-bounded and per-replica exec +// capabilities the bundle diagnostics collectors prefer (research R2). +var ( + _ collectBackend = (*DockerBackend)(nil) + _ contextExecer = (*DockerBackend)(nil) + _ replicaExecer = (*DockerBackend)(nil) + _ contextCopier = (*DockerBackend)(nil) +) + +// ExecContext is the timeout-bounded variant of Exec used by the bundle +// collectors (research R2: 60s per status dump). +func (d *DockerBackend) ExecContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) (string, error) { + return d.executor.runCommandContext(ctx, timeout, "docker", d.buildExecArgs(service, command, opts)...) +} + +// ExecReplica executes a command in one specific container by name, unlike +// Exec which targets a compose service (and therefore one arbitrary replica of +// a scaled service). The bundle collectors use it to capture per-replica +// task-worker state. +func (d *DockerBackend) ExecReplica(ctx context.Context, timeout time.Duration, replica Replica, command []string) (string, error) { + args := append([]string{"exec", replica.Container}, command...) + return d.executor.runCommandContext(ctx, timeout, "docker", args...) +} + +// composePSContainer is the subset of one `docker compose ps --format json` +// entry the collect primitives consume. +type composePSContainer struct { + Name string `json:"Name"` + Service string `json:"Service"` + State string `json:"State"` + Labels string `json:"Labels"` +} + +// parseComposePSContainers decodes `docker compose ps --format json` output. +// Recent Compose releases emit NDJSON (one object per line); older ones emit a +// single JSON array — both are accepted. One-off containers created by +// `docker compose run` are excluded, and container names are normalized +// without the leading slash some Docker APIs prepend. +func parseComposePSContainers(output string) ([]composePSContainer, error) { + trimmed := strings.TrimSpace(output) + if trimmed == "" { + return []composePSContainer{}, nil + } + + containers := []composePSContainer{} + if strings.HasPrefix(trimmed, "[") { + if err := json.Unmarshal([]byte(trimmed), &containers); err != nil { + return nil, fmt.Errorf("failed to parse docker compose ps output: %w", err) + } + } else { + for _, line := range nonEmptyLines(trimmed) { + var container composePSContainer + if err := json.Unmarshal([]byte(line), &container); err != nil { + return nil, fmt.Errorf("failed to parse docker compose ps line %q: %w", line, err) + } + containers = append(containers, container) + } + } + + kept := make([]composePSContainer, 0, len(containers)) + for _, container := range containers { + if strings.Contains(container.Labels, "com.docker.compose.oneoff=True") { + continue + } + container.Name = strings.TrimPrefix(container.Name, "/") + kept = append(kept, container) + } + return kept, nil +} + +// replicasFromComposePS converts compose ps entries for one service into +// Replicas sorted by container name: every replica of a scaled service +// appears, each carrying its human-meaningful container name (critique P2). +// Pod stays empty and Restarted false — previous logs do not exist on Docker. +func replicasFromComposePS(service string, containers []composePSContainer) []Replica { + replicas := []Replica{} + for _, container := range containers { + if container.Service != service || container.Name == "" { + continue + } + replicas = append(replicas, Replica{Service: service, Container: container.Name}) + } + sort.Slice(replicas, func(i, j int) bool { + return replicas[i].Container < replicas[j].Container + }) + return replicas +} + +// composePSContainers runs `docker compose ps --format json` (optionally +// including stopped containers) and parses the entries. stdout is read through +// the pipe primitive so compose warnings on stderr never pollute the JSON +// stream. +func (d *DockerBackend) composePSContainers(all bool, services ...string) ([]composePSContainer, error) { + psArgs := []string{"ps", "--format", "json"} + if all { + psArgs = append(psArgs, "-a") + } + psArgs = append(psArgs, services...) + + reader, wait, err := d.executor.runCommandPipeContext(context.Background(), collectExecTimeout, "docker", d.composeArgs(psArgs...)...) + if err != nil { + return nil, fmt.Errorf("failed to run docker compose ps: %w", err) + } + data, readErr := io.ReadAll(reader) + if waitErr := wait(); waitErr != nil { + return nil, fmt.Errorf("docker compose ps failed: %w", waitErr) + } + if readErr != nil { + return nil, fmt.Errorf("failed to read docker compose ps output: %w", readErr) + } + return parseComposePSContainers(string(data)) +} + +// ServiceReplicas enumerates the project's containers for one compose service. +// Stopped containers are included (`ps -a`): a deployed-but-stopped service +// must surface as a failed collector with its logs still collected, not vanish +// as skipped (FR-009). A service with no containers is a valid empty +// enumeration so callers record it as skipped. +func (d *DockerBackend) ServiceReplicas(service string) ([]Replica, error) { + containers, err := d.composePSContainers(true, service) + if err != nil { + return nil, fmt.Errorf("failed to list %s containers: %w", service, err) + } + return replicasFromComposePS(service, containers), nil +} + +// dockerLogArgs builds the docker logs arguments for one replica. +func dockerLogArgs(replica Replica, tailLines int) []string { + return []string{"logs", "--tail", strconv.Itoa(tailLines), replica.Container} +} + +// ReplicaLogs streams one container's logs via docker logs, bounded by the +// collect transfer timeout (research R2). The daemon demuxes container output +// onto stdout and stderr — Infrahub services log to stderr — so both streams +// are merged into the returned reader. Previous-container logs do not exist on +// Docker (Restarted is always false), so previous must never be requested. +func (d *DockerBackend) ReplicaLogs(replica Replica, tailLines int, previous bool) (io.ReadCloser, func() error, error) { + if previous { + return nil, nil, fmt.Errorf("previous container logs are not available on docker") + } + return d.executor.runCommandCombinedPipeContext(context.Background(), collectTransferTimeout, "docker", dockerLogArgs(replica, tailLines)...) +} + +// Metrics captures one-shot resource statistics for the project's running +// containers via `docker stats --no-stream` (research R4). Container names are +// part of the default stats columns, so support can map rows to replicas. +// Stopped containers are excluded — docker stats errors on them. +func (d *DockerBackend) Metrics() (string, error) { + containers, err := d.composePSContainers(false) + if err != nil { + return "", fmt.Errorf("failed to enumerate project containers: %w", err) + } + + names := []string{} + for _, container := range containers { + if container.Name != "" { + names = append(names, container.Name) + } + } + sort.Strings(names) + if len(names) == 0 { + return "", fmt.Errorf("no running containers found for project %s", d.project) + } + + args := append([]string{"stats", "--no-stream"}, names...) + output, err := d.executor.runCommandContext(context.Background(), collectExecTimeout, "docker", args...) + if err != nil { + if output != "" { + return "", fmt.Errorf("docker stats failed: %w: %s", err, output) + } + return "", fmt.Errorf("docker stats failed: %w", err) + } + return output, nil +} + +// dockerBenchmarkRunArgs builds the docker run arguments for the transient +// benchmark container, ported from the Python tool's collect_benchmark step +// (`docker run --pull always --rm `) plus an explicit name — so the +// timeout path can force-remove the container — and the project network +// attachment (research R11). +func dockerBenchmarkRunArgs(name, network, image string) []string { + args := []string{"run", "--pull", "always", "--rm", "--name", name} + if network != "" { + args = append(args, "--network", network) + } + return append(args, image) +} + +// pickComposeNetwork chooses the network the benchmark container joins: the +// compose project's default network when present, otherwise the first project +// network in lexical order, otherwise none (empty string). +func pickComposeNetwork(project string, networks []string) string { + if len(networks) == 0 { + return "" + } + preferred := project + "_default" + if contains(networks, preferred) { + return preferred + } + sorted := append([]string(nil), networks...) + sort.Strings(sorted) + return sorted[0] +} + +// benchmarkNetwork resolves the compose project's network for the benchmark +// container. Discovery failure degrades to no attachment (empty string): the +// benchmark measures host resources and does not require the project network. +func (d *DockerBackend) benchmarkNetwork() string { + output, err := d.executor.runCommandContext(context.Background(), collectExecTimeout, + "docker", "network", "ls", + "--filter", "label=com.docker.compose.project="+d.project, + "--format", "{{.Name}}") + if err != nil { + logrus.Debugf("Failed to enumerate networks of project %s: %v", d.project, err) + return "" + } + return pickComposeNetwork(d.project, nonEmptyLines(output)) +} + +// RunBenchmark runs the opt-in benchmark image as a transient container +// attached to the compose project's network (research R11) and returns its +// combined output. The container is force-removed afterwards even when the +// run timed out: --rm only covers a clean exit, and killing the docker CLI +// leaves the daemon-side container behind. Removing it is permitted — FR-010 +// protects the deployment's workloads and this container is the tool's own +// transient resource. +func (d *DockerBackend) RunBenchmark(ctx context.Context, image, containerName string) (string, error) { + defer func() { + // context.Background(): the container must be removed even when ctx + // was cancelled by a timeout or an interrupt. + if _, err := d.executor.runCommandContext(context.Background(), collectExecTimeout, + "docker", "rm", "-f", containerName); err != nil { + logrus.Debugf("Failed to remove benchmark container %s (already removed?): %v", containerName, err) + } + }() + + args := dockerBenchmarkRunArgs(containerName, d.benchmarkNetwork(), image) + output, err := d.executor.runCommandContext(ctx, collectBenchmarkTimeout, "docker", args...) + if err != nil { + if output != "" { + return "", fmt.Errorf("docker run failed: %w: %s", err, commandErrorLine(output)) + } + return "", fmt.Errorf("docker run failed: %w", err) + } + return output, nil +} + func (d *DockerBackend) ExecStreamPipe(service string, command []string, opts *ExecOptions) (io.ReadCloser, func() error, error) { return d.executor.runCommandPipe("docker", d.buildExecArgs(service, command, opts)...) } @@ -129,6 +383,19 @@ func (d *DockerBackend) CopyFrom(service, src, dest string) error { return nil } +// CopyFromContext is the timeout-bounded variant of CopyFrom used by the bundle +// collectors (research R2, FIX-1): a wedged container or daemon can accept a +// `docker compose cp` and never return, so the collect path must never call the +// unbounded CopyFrom. The shared CopyFrom is left untouched for the backup tool. +func (d *DockerBackend) CopyFromContext(ctx context.Context, timeout time.Duration, service, src, dest string) error { + source := fmt.Sprintf("%s:%s", service, src) + cmd := d.composeArgs("cp", source, dest) + if _, err := d.executor.runCommandContext(ctx, timeout, "docker", cmd...); err != nil { + return err + } + return nil +} + func (d *DockerBackend) Start(services ...string) error { if len(services) == 0 { return nil diff --git a/src/internal/app/environment_docker_test.go b/src/internal/app/environment_docker_test.go new file mode 100644 index 0000000..c24fc08 --- /dev/null +++ b/src/internal/app/environment_docker_test.go @@ -0,0 +1,166 @@ +package app + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseComposePSContainers(t *testing.T) { + tests := []struct { + name string + output string + want []composePSContainer + wantErr string + }{ + { + name: "empty output yields no containers", + output: "", + want: []composePSContainer{}, + }, + { + name: "whitespace-only output yields no containers", + output: "\n \n", + want: []composePSContainer{}, + }, + { + name: "NDJSON lines (compose >= 2.21)", + output: `{"Name":"proj-cache-1","Service":"cache","State":"running","Labels":"com.docker.compose.oneoff=False"} +{"Name":"proj-task-worker-2","Service":"task-worker","State":"running","Labels":"com.docker.compose.oneoff=False"} +`, + want: []composePSContainer{ + {Name: "proj-cache-1", Service: "cache", State: "running", Labels: "com.docker.compose.oneoff=False"}, + {Name: "proj-task-worker-2", Service: "task-worker", State: "running", Labels: "com.docker.compose.oneoff=False"}, + }, + }, + { + name: "JSON array (older compose releases)", + output: `[{"Name":"proj-cache-1","Service":"cache","State":"exited"}]`, + want: []composePSContainer{ + {Name: "proj-cache-1", Service: "cache", State: "exited"}, + }, + }, + { + name: "one-off run containers are excluded", + output: `{"Name":"proj-task-worker-1","Service":"task-worker","State":"running","Labels":"com.docker.compose.oneoff=False"} +{"Name":"proj-task-worker-run-ab12","Service":"task-worker","State":"exited","Labels":"com.docker.compose.oneoff=True"} +`, + want: []composePSContainer{ + {Name: "proj-task-worker-1", Service: "task-worker", State: "running", Labels: "com.docker.compose.oneoff=False"}, + }, + }, + { + name: "leading slash in the container name is stripped", + output: `{"Name":"/proj-database-1","Service":"database","State":"running"}`, + want: []composePSContainer{ + {Name: "proj-database-1", Service: "database", State: "running"}, + }, + }, + { + name: "invalid JSON line", + output: "not-json\n", + wantErr: "failed to parse docker compose ps line", + }, + { + name: "invalid JSON array", + output: `[{"Name":]`, + wantErr: "failed to parse docker compose ps output", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseComposePSContainers(tt.output) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("parseComposePSContainers(%q) succeeded, want error containing %q", tt.output, tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("error = %q, want it to contain %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("parseComposePSContainers(%q) failed: %v", tt.output, err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseComposePSContainers(%q) = %+v, want %+v", tt.output, got, tt.want) + } + }) + } +} + +func TestReplicasFromComposePS(t *testing.T) { + containers := []composePSContainer{ + {Name: "proj-task-worker-2", Service: "task-worker", State: "running"}, + {Name: "proj-cache-1", Service: "cache", State: "running"}, + {Name: "proj-task-worker-1", Service: "task-worker", State: "running"}, + {Name: "", Service: "task-worker", State: "running"}, // nameless entries are dropped + } + + tests := []struct { + name string + service string + want []Replica + }{ + { + name: "every replica of a scaled service appears, sorted by name", + service: "task-worker", + want: []Replica{ + {Service: "task-worker", Container: "proj-task-worker-1"}, + {Service: "task-worker", Container: "proj-task-worker-2"}, + }, + }, + { + name: "single-replica service", + service: "cache", + want: []Replica{ + {Service: "cache", Container: "proj-cache-1"}, + }, + }, + { + name: "service without containers yields an empty slice", + service: "task-manager-background-svc", + want: []Replica{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := replicasFromComposePS(tt.service, containers) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("replicasFromComposePS(%q) = %+v, want %+v", tt.service, got, tt.want) + } + for _, replica := range got { + if replica.Pod != "" || replica.Restarted { + t.Errorf("docker replica %+v must have empty Pod and Restarted=false", replica) + } + } + }) + } +} + +func TestDockerLogArgs(t *testing.T) { + replica := Replica{Service: "task-worker", Container: "proj-task-worker-1"} + + got := dockerLogArgs(replica, 500) + want := []string{"logs", "--tail", "500", "proj-task-worker-1"} + if !reflect.DeepEqual(got, want) { + t.Errorf("dockerLogArgs() = %v, want %v", got, want) + } +} + +func TestDockerReplicaLogs_PreviousIsUnavailable(t *testing.T) { + backend := NewDockerBackend(&Configuration{}, NewCommandExecutor()) + + reader, wait, err := backend.ReplicaLogs(Replica{Service: "cache", Container: "proj-cache-1"}, 100, true) + if err == nil { + t.Fatal("ReplicaLogs(previous=true) succeeded on docker, want error") + } + if reader != nil || wait != nil { + t.Error("ReplicaLogs(previous=true) returned a stream despite the error") + } + if !strings.Contains(err.Error(), "not available on docker") { + t.Errorf("error = %q, want it to mention previous logs are unavailable on docker", err) + } +} diff --git a/src/internal/app/environment_kubernetes.go b/src/internal/app/environment_kubernetes.go index fc69435..d49993f 100644 --- a/src/internal/app/environment_kubernetes.go +++ b/src/internal/app/environment_kubernetes.go @@ -1,14 +1,39 @@ package app import ( + "context" + "errors" "fmt" "io" + "sort" "strconv" "strings" + "time" "github.com/sirupsen/logrus" ) +// errNoPodsMatched is returned by GetAllPods when kubectl calls succeeded but +// no pod matched the service. It is distinct from a real kubectl/cluster +// failure so callers (ServiceReplicas) can record "service not deployed" +// (skipped) only for a genuine empty match, and surface API/RBAC/cluster +// errors as collector failures instead of masking them (FIX-2). +var errNoPodsMatched = errors.New("no pods matched the service") + +// podRunner executes a kubectl command and returns its trimmed output. The +// shared pod-resolution helpers (getPodForService, GetAllPods — used by the +// backup tool) pass the unbounded executor.runCommand; the collect primitives +// pass a timeout-bounded runner so a wedged API server cannot hang the bundle +// run before the exec/copy timeout even applies (research R2, FIX-5). +type podRunner func(name string, args ...string) (string, error) + +// boundedRunner returns a podRunner that bounds every kubectl call by timeout. +func (k *KubernetesBackend) boundedRunner(ctx context.Context, timeout time.Duration) podRunner { + return func(name string, args ...string) (string, error) { + return k.executor.runCommandContext(ctx, timeout, name, args...) + } +} + type KubernetesBackend struct { config *Configuration executor *CommandExecutor @@ -235,14 +260,31 @@ func (k *KubernetesBackend) getPodStatuses(service string) ([]string, error) { return statuses, nil } +// getPodForService resolves a single pod for a service using the unbounded +// executor. It is the shared helper used by the backup tool; its behavior is +// unchanged. func (k *KubernetesBackend) getPodForService(service string) (string, error) { + return k.getPodForServiceWith(k.executor.runCommand, service) +} + +// getPodForServiceContext is the timeout-bounded pod resolver the collect +// primitives use so a hung API server cannot stall the run before the exec/copy +// timeout applies (research R2, FIX-5). It shares getPodForServiceWith with the +// unbounded getPodForService, differing only in the runner. +func (k *KubernetesBackend) getPodForServiceContext(ctx context.Context, timeout time.Duration, service string) (string, error) { + return k.getPodForServiceWith(k.boundedRunner(ctx, timeout), service) +} + +// getPodForServiceWith resolves a single pod for a service via the supplied +// runner, caching the result. HA clusters resolve to the primary pod. +func (k *KubernetesBackend) getPodForServiceWith(run podRunner, service string) (string, error) { if pod, ok := k.podCache[service]; ok && pod != "" { return pod, nil } selectors := k.podSelectors(service) for _, selector := range selectors { - output, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") if err != nil { continue } @@ -250,7 +292,7 @@ func (k *KubernetesBackend) getPodForService(service string) (string, error) { if len(pods) > 0 { // If multiple pods found, try to find the primary (for HA clusters like CloudNativePG) if len(pods) > 1 { - if primary := k.findPrimaryPod(pods); primary != "" { + if primary := k.findPrimaryPod(run, pods); primary != "" { k.podCache[service] = primary return primary, nil } @@ -260,7 +302,7 @@ func (k *KubernetesBackend) getPodForService(service string) (string, error) { } } - output, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") if err != nil { return "", err } @@ -274,11 +316,28 @@ func (k *KubernetesBackend) getPodForService(service string) (string, error) { return "", fmt.Errorf("no pods found for service %s in namespace %s", service, k.namespace) } -// GetAllPods returns all pod names for a given service +// GetAllPods returns all pod names for a given service using the unbounded +// executor (shared helper used by the backup tool). It returns errNoPodsMatched +// only when kubectl calls succeeded but nothing matched, and the wrapped +// kubectl error when the cluster/API itself is unreachable (FIX-2). func (k *KubernetesBackend) GetAllPods(service string) ([]string, error) { + return k.getAllPodsWith(k.executor.runCommand, service) +} + +// getAllPodsContext is the timeout-bounded pod enumerator the collect primitives +// use (research R2, FIX-5). +func (k *KubernetesBackend) getAllPodsContext(ctx context.Context, timeout time.Duration, service string) ([]string, error) { + return k.getAllPodsWith(k.boundedRunner(ctx, timeout), service) +} + +// getAllPodsWith enumerates every pod backing a service via the supplied +// runner. A successful-but-empty match returns errNoPodsMatched; a failed +// fallback kubectl call returns the wrapped kubectl error so cluster failures +// are never mistaken for an undeployed service (FIX-2). +func (k *KubernetesBackend) getAllPodsWith(run podRunner, service string) ([]string, error) { selectors := k.podSelectors(service) for _, selector := range selectors { - output, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") if err != nil { continue } @@ -287,5 +346,239 @@ func (k *KubernetesBackend) GetAllPods(service string) ([]string, error) { return pods, nil } } - return nil, fmt.Errorf("no pods found for service %s", service) + + // Fallback: substring match on pod names, mirroring getPodForService. The + // Helm chart labels some pods with a different service value than the + // canonical name (e.g. infrahub-server pods carry infrahub/service=server), + // so label selectors alone would miss them. A failure here means the + // cluster/API is unreachable, not that the service is absent. + output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + if err != nil { + return nil, fmt.Errorf("failed to list pods in namespace %s: %w", k.namespace, err) + } + pods := []string{} + for _, name := range nonEmptyLines(output) { + if strings.Contains(name, service) { + pods = append(pods, name) + } + } + if len(pods) > 0 { + return pods, nil + } + + return nil, fmt.Errorf("no pods found for service %s in namespace %s: %w", service, k.namespace, errNoPodsMatched) +} + +// KubernetesBackend implements the collect-side primitives (spec +// 003-collect-tool, research R3/R4) and the optional timeout-bounded and +// per-replica exec capabilities the diagnostics collectors prefer. +var ( + _ collectBackend = (*KubernetesBackend)(nil) + _ contextExecer = (*KubernetesBackend)(nil) + _ replicaExecer = (*KubernetesBackend)(nil) + _ contextCopier = (*KubernetesBackend)(nil) +) + +// buildExecArgsContext resolves the pod under a bounded runner and constructs +// kubectl exec arguments, so the pod-resolution kubectl call is bounded too +// (FIX-5) rather than only the exec that follows. +func (k *KubernetesBackend) buildExecArgsContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) ([]string, error) { + pod, err := k.getPodForServiceContext(ctx, timeout, service) + if err != nil { + return nil, err + } + finalCmd := k.prepareCommand(command, opts) + args := []string{"exec", "-n", k.namespace, pod, "--"} + args = append(args, finalCmd...) + return args, nil +} + +// ExecContext is the timeout-bounded variant of Exec used by the bundle +// collectors (research R2: 60s per status dump). Both the pod resolution and +// the exec itself are bounded (FIX-5). +func (k *KubernetesBackend) ExecContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) (string, error) { + args, err := k.buildExecArgsContext(ctx, timeout, service, command, opts) + if err != nil { + return "", err + } + return k.executor.runCommandContext(ctx, timeout, "kubectl", args...) +} + +// CopyFromContext is the timeout-bounded variant of CopyFrom used by the bundle +// collectors (research R2, FIX-1). Pod resolution is bounded by the exec +// timeout (a quick metadata lookup) while the copy itself gets the supplied +// transfer timeout. The shared CopyFrom is left untouched for the backup tool. +func (k *KubernetesBackend) CopyFromContext(ctx context.Context, timeout time.Duration, service, src, dest string) error { + pod, err := k.getPodForServiceContext(ctx, collectExecTimeout, service) + if err != nil { + return err + } + source := fmt.Sprintf("%s/%s:%s", k.namespace, pod, src) + if _, err := k.executor.runCommandContext(ctx, timeout, "kubectl", "cp", source, dest); err != nil { + return err + } + return nil +} + +// ExecReplica executes a command in one specific replica (pod container), +// unlike Exec which resolves a single pod per service. The bundle collectors +// use it to capture per-replica task-worker state. +func (k *KubernetesBackend) ExecReplica(ctx context.Context, timeout time.Duration, replica Replica, command []string) (string, error) { + args := []string{"exec", "-n", k.namespace, replica.Pod, "-c", replica.Container, "--"} + args = append(args, command...) + return k.executor.runCommandContext(ctx, timeout, "kubectl", args...) +} + +// kubectlContainerStatusArgs builds the kubectl arguments that list one +// " " pair per line for a pod's containers. +func kubectlContainerStatusArgs(namespace, pod string) []string { + return []string{ + "get", "pod", pod, "-n", namespace, + "-o", "jsonpath={range .status.containerStatuses[*]}{.name}{\" \"}{.restartCount}{\"\\n\"}{end}", + } +} + +// parsePodContainerStatuses converts kubectlContainerStatusArgs output into +// one Replica per pod container, with Restarted derived from that container's +// restartCount (critique E2: restart counts live per container). +func parsePodContainerStatuses(service, pod, output string) ([]Replica, error) { + replicas := []Replica{} + for _, line := range nonEmptyLines(output) { + fields := strings.Fields(line) + if len(fields) != 2 { + return nil, fmt.Errorf("unexpected container status line %q for pod %s", line, pod) + } + restarts, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, fmt.Errorf("invalid restart count %q for container %s/%s: %w", fields[1], pod, fields[0], err) + } + replicas = append(replicas, Replica{ + Service: service, + Pod: pod, + Container: fields[0], + Restarted: restarts > 0, + }) + } + return replicas, nil +} + +// ServiceReplicas enumerates one Replica per container of every pod backing a +// service, sorted by pod then container. A service with no matching pods is a +// valid empty enumeration (nil, nil) so callers can record it as skipped +// rather than failed. +func (k *KubernetesBackend) ServiceReplicas(service string) ([]Replica, error) { + pods, err := k.getAllPodsContext(context.Background(), collectExecTimeout, service) + if err != nil { + if errors.Is(err, errNoPodsMatched) { + // kubectl succeeded but nothing matched: the service is genuinely + // not deployed in this namespace, so callers record it as skipped. + return nil, nil + } + // A real kubectl/cluster failure (API/RBAC/unreachable) must surface so + // the collector records failed with the reason instead of masquerading + // as "service not deployed" (FIX-2). + return nil, fmt.Errorf("failed to enumerate %s pods: %w", service, err) + } + + replicas := []Replica{} + for _, pod := range pods { + args := kubectlContainerStatusArgs(k.namespace, pod) + output, err := k.executor.runCommandContext(context.Background(), collectExecTimeout, "kubectl", args...) + if err != nil { + return nil, fmt.Errorf("failed to read container statuses for pod %s: %w", pod, err) + } + podReplicas, err := parsePodContainerStatuses(service, pod, output) + if err != nil { + return nil, err + } + replicas = append(replicas, podReplicas...) + } + + sort.Slice(replicas, func(i, j int) bool { + if replicas[i].Pod != replicas[j].Pod { + return replicas[i].Pod < replicas[j].Pod + } + return replicas[i].Container < replicas[j].Container + }) + return replicas, nil +} + +// kubectlLogArgs builds the kubectl logs arguments for one replica. +func kubectlLogArgs(namespace string, replica Replica, tailLines int, previous bool) []string { + args := []string{ + "logs", "-n", namespace, replica.Pod, + "-c", replica.Container, + fmt.Sprintf("--tail=%d", tailLines), + } + if previous { + args = append(args, "--previous") + } + return args +} + +// ReplicaLogs streams a replica's logs through kubectl, bounded by the +// collect transfer timeout (research R2) so a hung kubelet cannot stall the +// whole run. +func (k *KubernetesBackend) ReplicaLogs(replica Replica, tailLines int, previous bool) (io.ReadCloser, func() error, error) { + args := kubectlLogArgs(k.namespace, replica, tailLines, previous) + return k.executor.runCommandPipeContext(context.Background(), collectTransferTimeout, "kubectl", args...) +} + +// kubectlBenchmarkRunArgs builds the kubectl run arguments for the one-off +// benchmark pod (research R11): attached so its output is captured, never +// restarted, removed after the attach session completes, and bounded by +// --pod-running-timeout so a failing image pull surfaces as an error instead +// of waiting out the whole benchmark bound. +func kubectlBenchmarkRunArgs(namespace, name, image string) []string { + return []string{ + "run", name, + "-n", namespace, + "--image=" + image, + "--image-pull-policy=Always", + "--restart=Never", + "--attach", + "--rm", + "--quiet", + "--pod-running-timeout=5m", + } +} + +// RunBenchmark runs the opt-in benchmark image as a one-off attached pod in +// the namespace (research R11) and returns its combined output. The pod is +// deleted afterwards even when the run timed out — kubectl's --rm only covers +// a completed attach session. Deleting it is permitted: FR-010 protects the +// deployment's workloads and this pod is the tool's own transient resource. +func (k *KubernetesBackend) RunBenchmark(ctx context.Context, image, podName string) (string, error) { + defer func() { + // context.Background(): the pod must be deleted even when ctx was + // cancelled by a timeout or an interrupt. + if _, err := k.executor.runCommandContext(context.Background(), collectExecTimeout, + "kubectl", "delete", "pod", podName, "-n", k.namespace, "--ignore-not-found=true", "--now"); err != nil { + logrus.Debugf("Failed to delete benchmark pod %s: %v", podName, err) + } + }() + + args := kubectlBenchmarkRunArgs(k.namespace, podName, image) + output, err := k.executor.runCommandContext(ctx, collectBenchmarkTimeout, "kubectl", args...) + if err != nil { + if output != "" { + return "", fmt.Errorf("kubectl run failed: %w: %s", err, commandErrorLine(output)) + } + return "", fmt.Errorf("kubectl run failed: %w", err) + } + return output, nil +} + +// Metrics captures one-shot pod resource metrics via kubectl top. A missing +// metrics-server surfaces as a normal error so the metrics collector records +// failed in the manifest without aborting the run (research R4). +func (k *KubernetesBackend) Metrics() (string, error) { + output, err := k.executor.runCommandContext(context.Background(), collectExecTimeout, "kubectl", "top", "pods", "-n", k.namespace) + if err != nil { + if output != "" { + return "", fmt.Errorf("kubectl top pods failed: %w: %s", err, output) + } + return "", fmt.Errorf("kubectl top pods failed: %w", err) + } + return output, nil } diff --git a/src/internal/app/environment_kubernetes_test.go b/src/internal/app/environment_kubernetes_test.go new file mode 100644 index 0000000..892b602 --- /dev/null +++ b/src/internal/app/environment_kubernetes_test.go @@ -0,0 +1,61 @@ +package app + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// TestGetAllPodsWith_Branches covers the two failure modes GetAllPods must keep +// distinct (FIX-2): kubectl succeeds but nothing matches (errNoPodsMatched, so +// callers record "service not deployed"/skipped) versus a real cluster/API +// failure (a wrapped kubectl error, so callers record failed instead of masking +// the outage as an absent service). getAllPodsWith is the shared core of both +// the unbounded GetAllPods and the bounded getAllPodsContext. +func TestGetAllPodsWith_Branches(t *testing.T) { + k := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) + k.namespace = "infrahub" + + t.Run("a matching label selector returns the pods", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { + return "infrahub-server-0\ninfrahub-server-1\n", nil + } + pods, err := k.getAllPodsWith(run, "infrahub-server") + if err != nil { + t.Fatalf("getAllPodsWith failed: %v", err) + } + if len(pods) != 2 || pods[0] != "infrahub-server-0" || pods[1] != "infrahub-server-1" { + t.Errorf("pods = %v, want the two infrahub-server pods", pods) + } + }) + + t.Run("kubectl succeeds but nothing matches returns errNoPodsMatched", func(t *testing.T) { + // Every kubectl call succeeds and returns no pods. + run := func(name string, args ...string) (string, error) { return "", nil } + pods, err := k.getAllPodsWith(run, "task-manager-background-svc") + if pods != nil { + t.Errorf("pods = %v, want nil", pods) + } + if !errors.Is(err, errNoPodsMatched) { + t.Errorf("err = %v, want errNoPodsMatched so callers record skipped", err) + } + }) + + t.Run("a cluster/API failure surfaces a real error, not the sentinel", func(t *testing.T) { + // Label-selector calls and the fallback all fail (unreachable API/RBAC). + run := func(name string, args ...string) (string, error) { + return "The connection to the server was refused", fmt.Errorf("exit status 1") + } + _, err := k.getAllPodsWith(run, "infrahub-server") + if err == nil { + t.Fatal("getAllPodsWith succeeded despite an unreachable cluster, want error") + } + if errors.Is(err, errNoPodsMatched) { + t.Errorf("cluster failure was masked as errNoPodsMatched (would exit 0 with an empty bundle): %v", err) + } + if !strings.Contains(err.Error(), "failed to list pods") { + t.Errorf("err = %v, want the wrapped kubectl failure", err) + } + }) +} diff --git a/src/internal/app/kubernetes_utils.go b/src/internal/app/kubernetes_utils.go index ed7a1c5..d11674a 100644 --- a/src/internal/app/kubernetes_utils.go +++ b/src/internal/app/kubernetes_utils.go @@ -17,16 +17,19 @@ func (k *KubernetesBackend) podSelectors(service string) []string { } } -// findPrimaryPod searches for a pod with primary role label (for HA PostgreSQL clusters like CloudNativePG) -func (k *KubernetesBackend) findPrimaryPod(pods []string) string { +// findPrimaryPod searches for a pod with primary role label (for HA PostgreSQL +// clusters like CloudNativePG). The runner is supplied by the caller so the +// collect path can bound these lookups while the backup path keeps the +// unbounded executor (FIX-5). +func (k *KubernetesBackend) findPrimaryPod(run podRunner, pods []string) string { for _, pod := range pods { - output, err := k.executor.runCommand("kubectl", "get", "pod", pod, "-n", k.namespace, "-o", "jsonpath={.metadata.labels.cnpg\\.io/instanceRole}") + output, err := run("kubectl", "get", "pod", pod, "-n", k.namespace, "-o", "jsonpath={.metadata.labels.cnpg\\.io/instanceRole}") if err == nil && output == "primary" { logrus.Debugf("Found primary pod via cnpg.io/instanceRole: %s", pod) return pod } // Fallback to legacy role label - output, err = k.executor.runCommand("kubectl", "get", "pod", pod, "-n", k.namespace, "-o", "jsonpath={.metadata.labels.role}") + output, err = run("kubectl", "get", "pod", pod, "-n", k.namespace, "-o", "jsonpath={.metadata.labels.role}") if err == nil && output == "primary" { logrus.Debugf("Found primary pod via role label: %s", pod) return pod diff --git a/src/internal/app/masking.go b/src/internal/app/masking.go new file mode 100644 index 0000000..3e62d80 --- /dev/null +++ b/src/internal/app/masking.go @@ -0,0 +1,121 @@ +package app + +import ( + "encoding/json" + "regexp" + "strings" +) + +// maskedValue replaces sensitive values in env/config dumps before they are +// written into a troubleshooting bundle (research R5, FR-008). +const maskedValue = "********" + +// sensitiveKeySubstrings is the key-name match list: a key is sensitive when +// it contains any of these substrings, case-insensitively. The normative +// minimum is password|secret|token|key (research R5, FR-008); "pass" widens +// the match to subsume "password" and catch credential keys such as Redis +// "requirepass" and RabbitMQ "default_pass" — over-masking is safe, leaking +// is not. +var sensitiveKeySubstrings = []string{"pass", "secret", "token", "key"} + +// isSensitiveKey reports whether a key name refers to a sensitive value, +// using a case-insensitive substring match on the normative token list. +func isSensitiveKey(key string) bool { + lower := strings.ToLower(key) + for _, substring := range sensitiveKeySubstrings { + if strings.Contains(lower, substring) { + return true + } + } + return false +} + +// maskEnvOutput masks values in KEY=VALUE dumps (one pair per line), such as +// the output of `env` inside a container. Lines without an equals sign and +// values of non-sensitive keys pass through unchanged. Only the key part +// (before the first '=') is matched, so sensitive substrings inside values do +// not trigger masking. +func maskEnvOutput(input string) string { + lines := strings.Split(input, "\n") + for i, line := range lines { + idx := strings.Index(line, "=") + if idx < 0 { + continue + } + key := line[:idx] + if isSensitiveKey(key) { + lines[i] = key + "=" + maskedValue + } + } + return strings.Join(lines, "\n") +} + +// maskConfigPairs masks values in alternating key/value line dumps, the +// format produced by `redis-cli CONFIG GET '*'` in non-interactive mode: +// even lines carry the parameter name, odd lines carry its value. +func maskConfigPairs(input string) string { + lines := strings.Split(input, "\n") + for i := 0; i+1 < len(lines); i += 2 { + if isSensitiveKey(strings.TrimSpace(lines[i])) { + lines[i+1] = maskedValue + } + } + return strings.Join(lines, "\n") +} + +// erlangTuplePattern matches flat two-element Erlang tuples such as +// {default_pass,<<"guest">>} in `rabbitmqctl environment` / `rabbitmqctl +// status` output. Nested tuples/lists as values are not matched; scalar +// credentials in the RabbitMQ configuration are flat tuples. +var erlangTuplePattern = regexp.MustCompile(`\{\s*'?([A-Za-z0-9_.]+)'?\s*,([^{}\[\]]*)\}`) + +// maskErlangConfig masks values of sensitive keys in Erlang proplist-style +// dumps (`rabbitmqctl environment` and `rabbitmqctl status`). +func maskErlangConfig(input string) string { + return erlangTuplePattern.ReplaceAllStringFunc(input, func(match string) string { + submatch := erlangTuplePattern.FindStringSubmatch(match) + if submatch == nil || !isSensitiveKey(submatch[1]) { + return match + } + return "{" + submatch[1] + "," + maskedValue + "}" + }) +} + +// maskJSON masks values of sensitive keys anywhere in a JSON document, such +// as the server API configuration dump (research R5). Input that is not valid +// JSON is returned unchanged so callers can store error responses as-is and +// keep failures observable. +func maskJSON(input string) string { + var document any + if err := json.Unmarshal([]byte(input), &document); err != nil { + return input + } + masked, err := json.MarshalIndent(maskJSONValue(document), "", " ") + if err != nil { + return input + } + return string(masked) +} + +// maskJSONValue recursively masks sensitive-keyed values in a decoded JSON +// tree. The whole value of a sensitive key is replaced, whatever its type. +func maskJSONValue(value any) any { + switch typed := value.(type) { + case map[string]any: + for key, nested := range typed { + if isSensitiveKey(key) { + typed[key] = maskedValue + continue + } + typed[key] = maskJSONValue(nested) + } + return typed + case []any: + for i, item := range typed { + typed[i] = maskJSONValue(item) + } + return typed + default: + return value + } +} diff --git a/src/internal/app/masking_test.go b/src/internal/app/masking_test.go new file mode 100644 index 0000000..cafce45 --- /dev/null +++ b/src/internal/app/masking_test.go @@ -0,0 +1,255 @@ +package app + +import "testing" + +func TestIsSensitiveKey(t *testing.T) { + tests := []struct { + name string + key string + want bool + }{ + {"exact password", "password", true}, + {"upper case password", "PASSWORD", true}, + {"mixed case token", "MyToken", true}, + {"substring password", "NEO4J_PASSWORD", true}, + {"substring secret", "client_secret_value", true}, + {"substring key", "api_key", true}, + {"substring key upper", "AWS_SECRET_ACCESS_KEY", true}, + {"exact secret", "SECRET", true}, + {"exact token upper", "TOKEN", true}, + {"plain username", "username", false}, + {"plain host", "INFRAHUB_HOST", false}, + {"empty key", "", false}, + {"pass substring matches", "default_pass", true}, + {"redis requirepass matches", "requirepass", true}, + {"passphrase matches", "SSL_PASSPHRASE", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isSensitiveKey(tt.key); got != tt.want { + t.Errorf("isSensitiveKey(%q) = %v, want %v", tt.key, got, tt.want) + } + }) + } +} + +func TestMaskEnvOutput(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "sensitive key masked", + input: "NEO4J_PASSWORD=admin123", + want: "NEO4J_PASSWORD=" + maskedValue, + }, + { + name: "non-sensitive key untouched", + input: "INFRAHUB_HOST=localhost", + want: "INFRAHUB_HOST=localhost", + }, + { + name: "case-insensitive match", + input: "db_password=hunter2", + want: "db_password=" + maskedValue, + }, + { + name: "empty value still masked", + input: "API_TOKEN=", + want: "API_TOKEN=" + maskedValue, + }, + { + name: "value containing equals fully masked", + input: "SECRET_URI=postgres://u:p@host?sslmode=verify", + want: "SECRET_URI=" + maskedValue, + }, + { + name: "sensitive substring in value only does not mask", + input: "GREETING=my password is safe", + want: "GREETING=my password is safe", + }, + { + name: "line without equals untouched", + input: "not-an-assignment", + want: "not-an-assignment", + }, + { + name: "empty input", + input: "", + want: "", + }, + { + name: "multi-line mixed", + input: "PATH=/usr/bin\nAWS_SECRET_ACCESS_KEY=abc\nHOME=/root\nJWT_TOKEN=xyz\n", + want: "PATH=/usr/bin\nAWS_SECRET_ACCESS_KEY=" + maskedValue + "\nHOME=/root\nJWT_TOKEN=" + maskedValue + "\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maskEnvOutput(tt.input); got != tt.want { + t.Errorf("maskEnvOutput(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestMaskConfigPairs(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "sensitive pair masked", + input: "masterauth-token\nsupersecretvalue", + want: "masterauth-token\n" + maskedValue, + }, + { + name: "non-sensitive pair untouched", + input: "maxmemory\n0", + want: "maxmemory\n0", + }, + { + name: "case-insensitive key match", + input: "TLS-KEY-FILE\n/etc/redis/tls.key", + want: "TLS-KEY-FILE\n" + maskedValue, + }, + { + name: "empty value still masked", + input: "auth-token\n", + want: "auth-token\n" + maskedValue, + }, + { + name: "redis requirepass masked", + input: "requirepass\nhunter2", + want: "requirepass\n" + maskedValue, + }, + { + name: "mixed pairs", + input: "maxmemory\n100mb\naccess-token\nabc123\nappendonly\nno", + want: "maxmemory\n100mb\naccess-token\n" + maskedValue + "\nappendonly\nno", + }, + { + name: "odd trailing key line untouched", + input: "maxmemory\n0\ndangling-token", + want: "maxmemory\n0\ndangling-token", + }, + { + name: "empty input", + input: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maskConfigPairs(tt.input); got != tt.want { + t.Errorf("maskConfigPairs(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestMaskErlangConfig(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "sensitive tuple masked", + input: `{cookie_secret,<<"abc123">>},`, + want: "{cookie_secret," + maskedValue + "},", + }, + { + name: "rabbitmq default_pass masked", + input: `{default_pass,<<"guest">>}`, + want: "{default_pass," + maskedValue + "}", + }, + { + name: "non-sensitive tuple untouched", + input: `{default_user,<<"guest">>},`, + want: `{default_user,<<"guest">>},`, + }, + { + name: "case-insensitive key match", + input: `{AUTH_TOKEN,<<"xyz">>}`, + want: "{AUTH_TOKEN," + maskedValue + "}", + }, + { + name: "empty value still masked", + input: "{ssl_key,}", + want: "{ssl_key," + maskedValue + "}", + }, + { + name: "quoted atom key masked", + input: `{'secret_backend',classic}`, + want: "{secret_backend," + maskedValue + "}", + }, + { + name: "list-valued tuple untouched, flat sensitive tuple masked", + input: `[{rabbit,[{auth_backends,[rabbit_auth_backend_internal]},{oauth_token_scope,<<"tag">>}]}]`, + want: "[{rabbit,[{auth_backends,[rabbit_auth_backend_internal]},{oauth_token_scope," + maskedValue + "}]}]", + }, + { + name: "empty input", + input: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maskErlangConfig(tt.input); got != tt.want { + t.Errorf("maskErlangConfig(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestMaskJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "top-level sensitive key masked", + input: `{"username":"admin","password":"hunter2"}`, + want: "{\n \"password\": \"" + maskedValue + "\",\n \"username\": \"admin\"\n}", + }, + { + name: "nested objects and arrays walked", + input: `{"security":{"secret_key":"abc"},"servers":[{"api_token":"t"},{"host":"h"}]}`, + want: "{\n \"security\": {\n \"secret_key\": \"" + maskedValue + "\"\n },\n" + + " \"servers\": [\n {\n \"api_token\": \"" + maskedValue + "\"\n },\n" + + " {\n \"host\": \"h\"\n }\n ]\n}", + }, + { + name: "sensitive key with object value fully replaced", + input: `{"secrets":{"inner":"value"}}`, + want: "{\n \"secrets\": \"" + maskedValue + "\"\n}", + }, + { + name: "non-JSON input returned unchanged", + input: "HTTP 502 Bad Gateway", + want: "HTTP 502 Bad Gateway", + }, + { + name: "empty input returned unchanged", + input: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maskJSON(tt.input); got != tt.want { + t.Errorf("maskJSON(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} diff --git a/src/internal/app/scripts/collect_prefect_events.py b/src/internal/app/scripts/collect_prefect_events.py new file mode 100644 index 0000000..16608ac --- /dev/null +++ b/src/internal/app/scripts/collect_prefect_events.py @@ -0,0 +1,63 @@ +"""Dump recent Prefect events as JSON for the troubleshooting bundle. + +The Prefect CLI has no non-streaming events command, so this script queries +the server's events API directly. It runs inside the task-manager container, +which ships httpx as a Prefect dependency. The events endpoint caps the page +size at 50, so recent events are gathered by following next_page links. +""" + +import json +import os +import sys + +import httpx + +DEFAULT_API_URL = "http://localhost:4200/api" +PAGE_SIZE = 50 +EVENT_LIMIT = 200 +# Hard cap on pages followed. With PAGE_SIZE=50 and EVENT_LIMIT=200 four pages +# suffice; the bound stops a runaway loop if the API keeps returning a truthy +# next_page (FIX-9). +MAX_PAGES = 20 + + +def main() -> int: + api_url = os.environ.get("PREFECT_API_URL", DEFAULT_API_URL).rstrip("/") + + events = [] + total = 0 + with httpx.Client(timeout=30) as client: + response = client.post( + f"{api_url}/events/filter", + json={"limit": PAGE_SIZE}, + ) + response.raise_for_status() + page = response.json() + total = page.get("total", 0) + events.extend(page.get("events", [])) + + pages = 1 + while len(events) < EVENT_LIMIT and page.get("next_page") and pages < MAX_PAGES: + response = client.get(page["next_page"]) + response.raise_for_status() + page = response.json() + new_events = page.get("events", []) + # A truthy next_page pointing at an empty page would otherwise spin + # forever; stop as soon as a page yields nothing (FIX-9). + if not new_events: + break + events.extend(new_events) + pages += 1 + + json.dump( + {"total": total, "collected": len(events), "events": events[:EVENT_LIMIT]}, + sys.stdout, + indent=2, + default=str, + ) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 19183c9..ebdf80c 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -73,6 +73,18 @@ def backup_binary() -> str: return str(binary) +# --------------------------------------------------------------------------- +# Fixture: collect_binary +# --------------------------------------------------------------------------- +@pytest.fixture(scope="session") +def collect_binary() -> str: + """Build infrahub-collect and return the binary path.""" + subprocess.run(["make", "build"], cwd=str(PROJECT_ROOT), check=True) + binary = PROJECT_ROOT / "bin" / "infrahub-collect" + assert binary.exists(), f"Binary not found at {binary}" + return str(binary) + + # --------------------------------------------------------------------------- # Fixture: minio_docker (testcontainers) # --------------------------------------------------------------------------- diff --git a/tests/e2e/test_docker_collect.py b/tests/e2e/test_docker_collect.py new file mode 100644 index 0000000..4ac1d29 --- /dev/null +++ b/tests/e2e/test_docker_collect.py @@ -0,0 +1,367 @@ +"""E2E tests: Docker Compose + troubleshooting bundle collection. + +Covers spec 003-collect-tool US2 (quickstart scenarios 3, 4, and 6): full +bundle with parity content and a schema-validated manifest, project selection +with and without --project, the degraded case (stopped cache container → +exit 0 + failed manifest entry, FR-009/SC-005), masked env output (FR-008), +--log-lines / INFRAHUB_LOG_LINES precedence (FR-011), and US3 +(--include-backup: standard backup next to the bundle, referenced in the +manifest, FR-014). + +The tests share one class-scoped compose stack; the disruptive scenarios run +last so the stack stays healthy for the earlier ones: the degraded-cache test +stops/starts the cache container, and the include-backup test — the very last +— stops and restarts application containers through the inherited backup +behavior. +""" + +import json +import os +import re +import subprocess +import tarfile +from pathlib import Path + +import pytest +from infrahub_sdk.testing.docker import TestInfrahubDockerClient +from infrahub_testcontainers.container import InfrahubDockerCompose + +from tests.helpers.bundle import ( + OPT_IN_COLLECTORS, + assert_bundle_layout, + assert_collector_outcomes, + find_bundle, + read_bundle_archive, + read_bundle_member, + validate_manifest_schema, +) +from tests.helpers.utils import find_latest_backup, run_collect, wait_for_http + +# The compose stack scales task-worker to 2 so per-replica collection on a +# scaled compose service is observable (FR-002). +TASK_WORKER_COUNT = 2 + +# Mirrors the tool's masking contract (research R5): keys matching these +# substrings must have masked values in env dumps. +SENSITIVE_KEY_RE = re.compile(r"pass|secret|token|key", re.IGNORECASE) +ENV_VAR_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +MASKED_VALUE = "********" + + +# --------------------------------------------------------------------------- +# docker compose helpers +# --------------------------------------------------------------------------- +def _compose(project: str, *args: str, check: bool = True) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", "compose", "-p", project, *args], + capture_output=True, + text=True, + ) + if check and result.returncode != 0: + raise RuntimeError(f"docker compose {' '.join(args)} failed (exit {result.returncode}): {result.stderr}") + return result + + +def _service_container_names(project: str, service: str) -> list[str]: + """Container names backing one compose service (running or stopped).""" + output = _compose(project, "ps", "-a", "--format", "json", service).stdout + names = [] + for line in output.splitlines(): + line = line.strip() + if not line: + continue + entry = json.loads(line) + if entry.get("Service") == service and entry.get("Name"): + names.append(entry["Name"]) + return sorted(names) + + +def _list_infrahub_compose_projects() -> list[str]: + """Enumerate Infrahub compose projects the way ListDockerProjects does.""" + output = subprocess.run(["docker", "compose", "ls"], capture_output=True, text=True, check=True).stdout + projects = [] + for line in output.splitlines(): + line = line.strip() + if not line or line.upper().startswith("NAME "): + continue + name = line.split()[0] + ps = subprocess.run(["docker", "compose", "-p", name, "ps", "-a"], capture_output=True, text=True) + if ps.returncode == 0 and "infrahub" in ps.stdout.lower(): + projects.append(name) + return sorted(projects) + + +def _assert_env_dump_masked(env_text: str, plaintext_secrets: list[str]) -> None: + """FR-008: sensitive keys masked, known secret values absent.""" + masked = 0 + for line in env_text.splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + if not ENV_VAR_NAME_RE.match(key): + continue # continuation line of a multi-line value + if SENSITIVE_KEY_RE.search(key): + assert value == MASKED_VALUE, f"Sensitive env var {key} not masked: {value!r}" + masked += 1 + assert masked > 0, "env dump contains no masked entries; masking cannot be verified" + for secret in plaintext_secrets: + assert secret not in env_text, f"Plaintext secret {secret!r} leaked into the env dump" + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- +@pytest.mark.e2e +@pytest.mark.docker +class TestDockerCollect(TestInfrahubDockerClient): + @pytest.fixture(scope="class") + def infrahub_compose( + self, + request: pytest.FixtureRequest, + tmp_directory: Path, + remote_repos_dir: Path, + remote_backups_dir: Path, + infrahub_version: str, + deployment_type: str | None, + ) -> InfrahubDockerCompose: + """Compose project with task-worker scaled to TASK_WORKER_COUNT. + + Overrides the session default of 1 worker (tests/e2e/conftest.py) so + per-replica collection on a scaled compose service is observable. + Compose interpolation reads the process environment when `up` runs + (it overrides the generated .env), so the variable must stay set for + the whole class and is restored at class teardown. + """ + original = os.environ.get("INFRAHUB_TESTING_TASK_WORKER_COUNT") + + def restore() -> None: + if original is None: + os.environ.pop("INFRAHUB_TESTING_TASK_WORKER_COUNT", None) + else: + os.environ["INFRAHUB_TESTING_TASK_WORKER_COUNT"] = original + + request.addfinalizer(restore) + os.environ["INFRAHUB_TESTING_TASK_WORKER_COUNT"] = str(TASK_WORKER_COUNT) + return InfrahubDockerCompose.init( + directory=tmp_directory, + version=infrahub_version, + deployment_type=deployment_type, + ) + + async def test_collect_full_bundle(self, infrahub_compose, infrahub_port, collect_binary, tmp_path): + """US2 scenario 1: parity bundle with schema-validated manifest (FR-006/FR-008).""" + project = infrahub_compose.project_name + output_dir = tmp_path / "bundles" + + run_collect( + collect_binary, + ["--project", project, "--output-dir", str(output_dir), "create"], + ) + + # Archive integrity: valid tar.gz, everything under bundle/ + bundle_path = find_bundle(output_dir) + file_names, manifest = read_bundle_archive(bundle_path) + + # Manifest conforms to contracts/manifest.schema.json + validate_manifest_schema(manifest) + assert manifest["environment"] == "docker" + assert manifest["log_lines"] == 100000 # documented default + assert f"support_bundle_{manifest['collect_id']}.tar.gz" == bundle_path.name + + # SC-005: one manifest entry per planned collector, no duplicates + statuses = assert_collector_outcomes(manifest) + + # Parity content (US2 scenario 1): every collector succeeds on a + # healthy stack except the optional service that is scaled to zero + # and the opt-in extras, which are skipped when not requested. + assert statuses["logs/task-manager-background-svc"] == "skipped" + failed = {name: status for name, status in statuses.items() if status != "success"} + failed.pop("logs/task-manager-background-svc") + for extra in OPT_IN_COLLECTORS: + assert statuses[extra] == "skipped", f"Opt-in collector {extra} must be skipped when not requested" + failed.pop(extra) + assert not failed, f"Collectors did not succeed on a healthy stack: {failed}" + + # Layout per contracts/bundle-layout.md — identical to Kubernetes (SC-004) + assert_bundle_layout(file_names, statuses) + + # FR-002: one log file per replica of the scaled task-worker service, + # named after the container + worker_containers = _service_container_names(project, "task-worker") + assert len(worker_containers) >= TASK_WORKER_COUNT, ( + f"Expected >= {TASK_WORKER_COUNT} task-worker containers, found {worker_containers}" + ) + worker_logs = { + name.removeprefix("bundle/logs/task-worker/") + for name in file_names + if name.startswith("bundle/logs/task-worker/") + } + expected_worker_logs = {f"{name}.log" for name in worker_containers} + assert expected_worker_logs <= worker_logs, ( + f"Missing per-replica task-worker logs: {expected_worker_logs - worker_logs}" + ) + + # Previous-container logs do not exist on Docker + previous_logs = [name for name in file_names if name.endswith(".previous.log")] + assert not previous_logs, f"Unexpected previous logs in a Docker bundle: {previous_logs}" + + # Per-replica task-worker state directories, named after the container + for container in worker_containers: + assert any(name.startswith(f"bundle/task-worker/{container}/") for name in file_names), ( + f"Missing per-replica task-worker state for {container}" + ) + + # FR-008: masked env output contains no plaintext secrets + env_text = read_bundle_member(bundle_path, "bundle/server/environment.txt") + secrets = [ + infrahub_compose.get_env_var("INFRAHUB_TESTING_INITIAL_ADMIN_TOKEN"), + infrahub_compose.get_env_var("INFRAHUB_TESTING_INITIAL_AGENT_TOKEN"), + infrahub_compose.get_env_var("INFRAHUB_TESTING_SECURITY_SECRET_KEY"), + ] + _assert_env_dump_masked(env_text, secrets) + + async def test_collect_project_selection_without_flag( + self, infrahub_compose, infrahub_port, collect_binary, tmp_path + ): + """US2 scenario 2: without --project, selection behaves like the backup tool.""" + project = infrahub_compose.project_name + output_dir = tmp_path / "bundles" + + infrahub_projects = _list_infrahub_compose_projects() + assert project in infrahub_projects, f"Test project {project} not discoverable: {infrahub_projects}" + + if len(infrahub_projects) == 1: + # Exactly one Infrahub project on this host: auto-detection selects it. + run_collect(collect_binary, ["--output-dir", str(output_dir), "create"]) + _, manifest = read_bundle_archive(find_bundle(output_dir)) + assert manifest["environment"] == "docker" + else: + # Shared host with several Infrahub projects: the tool must refuse + # and demand --project, exactly like infrahub-backup. + result = subprocess.run( + [collect_binary, "--output-dir", str(output_dir), "create"], + capture_output=True, + text=True, + ) + assert result.returncode != 0, "create without --project succeeded despite multiple projects" + combined = result.stdout + result.stderr + assert "multiple docker compose projects found" in combined, ( + f"Expected project-selection error, got:\n{combined}" + ) + + async def test_collect_log_lines_flag_and_env(self, infrahub_compose, infrahub_port, collect_binary, tmp_path): + """FR-011 / quickstart scenario 6: --log-lines wins over INFRAHUB_LOG_LINES.""" + project = infrahub_compose.project_name + + flag_dir = tmp_path / "flag" + run_collect( + collect_binary, + ["--project", project, "--output-dir", str(flag_dir), "create", "--log-lines", "500"], + env={"INFRAHUB_LOG_LINES": "250"}, + ) + _, manifest = read_bundle_archive(find_bundle(flag_dir)) + assert manifest["log_lines"] == 500, "flag must win over the environment variable" + + env_dir = tmp_path / "env" + run_collect( + collect_binary, + ["--project", project, "--output-dir", str(env_dir), "create"], + env={"INFRAHUB_LOG_LINES": "250"}, + ) + _, manifest = read_bundle_archive(find_bundle(env_dir)) + assert manifest["log_lines"] == 250, "environment variable must apply when the flag is absent" + + async def test_collect_degraded_cache(self, infrahub_compose, infrahub_port, collect_binary, tmp_path): + """FR-009/SC-005 (quickstart scenario 4): stopped cache → exit 0 + failed entry. + + Runs after the healthy-stack scenarios: it degrades the shared stack + and restores it afterwards. + """ + project = infrahub_compose.project_name + output_dir = tmp_path / "bundles" + + _compose(project, "stop", "cache") + try: + # run_collect raises on a non-zero exit, so returning proves exit 0 + run_collect( + collect_binary, + ["--project", project, "--output-dir", str(output_dir), "create"], + ) + finally: + _compose(project, "start", "cache") + + bundle_path = find_bundle(output_dir) + file_names, manifest = read_bundle_archive(bundle_path) + validate_manifest_schema(manifest) + + # Every planned collector is still accounted for (SC-005) + statuses = assert_collector_outcomes(manifest) + + entries = {entry["name"]: entry for entry in manifest["collectors"]} + assert entries["cache-status"]["status"] == "failed", ( + f"cache-status = {entries['cache-status']}, want failed while cache is stopped" + ) + assert entries["cache-status"]["reason"], "failed cache-status entry must carry a reason" + + # A deployed-but-stopped container still yields its logs (FR-009) + assert statuses["logs/cache"] == "success", "logs of the stopped cache container must still be collected" + assert any(name.startswith("bundle/logs/cache/") for name in file_names) + + async def test_collect_include_backup(self, infrahub_compose, infrahub_port, collect_binary, tmp_path): + """US3 scenario 1 (FR-014): --include-backup creates a standard backup + next to the bundle and records it in the manifest. + + Runs last in the class: the delegated backup follows the standard + backup behavior, which stops and restarts application containers. + """ + project = infrahub_compose.project_name + output_dir = tmp_path / "bundles" + backup_dir = tmp_path / "backups" + + # run_collect raises on a non-zero exit, so returning proves exit 0 + run_collect( + collect_binary, + [ + "--project", + project, + "--output-dir", + str(output_dir), + "--backup-dir", + str(backup_dir), + "create", + "--include-backup", + ], + ) + + try: + # The backup artifact is a standard backup archive next to the + # bundle (not embedded in it), carrying the backup metadata. + backup_file = find_latest_backup(backup_dir) + with tarfile.open(backup_file, "r:gz") as tar: + metadata_file = tar.extractfile("backup/backup_information.json") + assert metadata_file is not None, "backup archive is missing backup_information.json" + backup_metadata = json.loads(metadata_file.read()) + assert backup_metadata["backup_id"] == backup_file.name.removesuffix(".tar.gz"), ( + f"Backup metadata id {backup_metadata['backup_id']!r} does not match {backup_file.name}" + ) + + # The manifest references the backup; the rest of the plan still ran + bundle_path = find_bundle(output_dir) + file_names, manifest = read_bundle_archive(bundle_path) + validate_manifest_schema(manifest) + statuses = assert_collector_outcomes(manifest) + assert statuses["backup"] == "success", f"backup collector = {statuses['backup']}, want success" + + entries = {entry["name"]: entry for entry in manifest["collectors"]} + assert entries["backup"].get("artifact") == str(backup_file), ( + f"Manifest backup artifact {entries['backup'].get('artifact')!r} " + f"does not reference the produced backup {backup_file}" + ) + + # The backup stays a sibling artifact, never embedded in the bundle + embedded_archives = [name for name in file_names if name.endswith(".tar.gz")] + assert not embedded_archives, f"Backup archive embedded in the bundle: {embedded_archives}" + finally: + # The inherited backup behavior stopped/restarted app containers; + # wait for recovery so class teardown sees a healthy deployment. + await wait_for_http(f"http://localhost:{infrahub_port}/api/config", timeout=180.0, interval=5.0) diff --git a/tests/e2e/test_k8s_collect.py b/tests/e2e/test_k8s_collect.py new file mode 100644 index 0000000..99a65f6 --- /dev/null +++ b/tests/e2e/test_k8s_collect.py @@ -0,0 +1,289 @@ +"""E2E tests: Kubernetes (vcluster) + troubleshooting bundle collection. + +Covers spec 003-collect-tool US1 (quickstart scenario 5): archive integrity, +manifest schema conformance, one log file per replica (task-worker scaled to +2), previous-container logs for an induced restart, bundle layout per +contracts/bundle-layout.md, and read-only collection (SC-003: zero pod +restarts or scale events caused by the run). +""" + +import json +import subprocess +import time + +import pytest + +from tests.helpers.bundle import ( + assert_bundle_layout, + assert_collector_outcomes, + find_bundle, + read_bundle_archive, + validate_manifest_schema, +) +from tests.helpers.utils import run_collect + + +# --------------------------------------------------------------------------- +# kubectl helpers +# --------------------------------------------------------------------------- +def _kubectl(kubeconfig: str, *args: str) -> str: + """Run kubectl against the test cluster and return stdout.""" + result = subprocess.run( + ["kubectl", "--kubeconfig", kubeconfig, *args], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"kubectl {' '.join(args)} failed (exit {result.returncode}): {result.stderr}") + return result.stdout + + +def _service_pods(kubeconfig: str, namespace: str, service: str) -> list[dict]: + """Return the pod objects backing an Infrahub service. + + Mirrors the collect backend's resolution order: the infrahub/service label + first, then a pod-name substring match (the Helm chart labels some pods + with a different value than the canonical service name). + """ + output = _kubectl( + kubeconfig, + "get", + "pods", + "-n", + namespace, + "-l", + f"infrahub/service={service}", + "-o", + "json", + ) + items = json.loads(output)["items"] + if items: + return items + all_pods = json.loads(_kubectl(kubeconfig, "get", "pods", "-n", namespace, "-o", "json"))["items"] + return [pod for pod in all_pods if service in pod["metadata"]["name"]] + + +def _find_task_worker_deployment(kubeconfig: str, namespace: str) -> str: + """Resolve the task-worker deployment name (label selector, then name match).""" + output = _kubectl( + kubeconfig, + "get", + "deployments", + "-n", + namespace, + "-l", + "infrahub/service=task-worker", + "-o", + "jsonpath={.items[*].metadata.name}", + ) + names = output.split() + if not names: + output = _kubectl(kubeconfig, "get", "deployments", "-n", namespace, "-o", "jsonpath={.items[*].metadata.name}") + names = [name for name in output.split() if "task-worker" in name] + assert names, "No task-worker deployment found" + return names[0] + + +def _scale_deployment(kubeconfig: str, namespace: str, deployment: str, replicas: int) -> None: + """Scale a deployment and wait for the rollout to settle.""" + _kubectl(kubeconfig, "scale", "deployment", deployment, "-n", namespace, f"--replicas={replicas}") + _kubectl(kubeconfig, "rollout", "status", f"deployment/{deployment}", "-n", namespace, "--timeout=300s") + + +def _container_restart_count(kubeconfig: str, namespace: str, pod: str, container: str) -> int: + output = _kubectl(kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "json") + for status in json.loads(output)["status"].get("containerStatuses", []): + if status["name"] == container: + return status["restartCount"] + raise RuntimeError(f"Container {container} not found in pod {pod}") + + +def _induce_container_restart(kubeconfig: str, namespace: str, pod: str, container: str) -> None: + """Restart a container in place (same pod) so previous logs become available. + + Signals PID 1 inside the container; the pod is NOT deleted, so the + container's restartCount increments and `kubectl logs --previous` works. + """ + baseline = _container_restart_count(kubeconfig, namespace, pod, container) + for signal_name in ("TERM", "INT", "KILL"): + subprocess.run( + [ + "kubectl", + "--kubeconfig", + kubeconfig, + "exec", + "-n", + namespace, + pod, + "-c", + container, + "--", + "sh", + "-c", + f"kill -{signal_name} 1", + ], + capture_output=True, + text=True, + ) + deadline = time.time() + 90 + while time.time() < deadline: + if _container_restart_count(kubeconfig, namespace, pod, container) > baseline: + return + time.sleep(3) + pytest.fail(f"Could not induce a restart of container {container} in pod {pod}") + + +def _wait_pods_ready(kubeconfig: str, namespace: str, service: str, timeout: int = 300) -> None: + """Wait until every pod of a service is Ready.""" + _kubectl( + kubeconfig, + "wait", + "--for=condition=Ready", + "pod", + "-n", + namespace, + "-l", + f"infrahub/service={service}", + f"--timeout={timeout}s", + ) + + +def _cluster_state(kubeconfig: str, namespace: str) -> dict: + """Snapshot per-container restart counts, pod identities, and workload replicas (SC-003).""" + pods = json.loads(_kubectl(kubeconfig, "get", "pods", "-n", namespace, "-o", "json"))["items"] + restarts = {} + pod_uids = set() + for pod in pods: + name = pod["metadata"]["name"] + pod_uids.add(pod["metadata"]["uid"]) + for status in pod.get("status", {}).get("containerStatuses", []): + restarts[f"{name}/{status['name']}"] = status["restartCount"] + replicas = {} + for kind in ("deployment", "statefulset"): + items = json.loads(_kubectl(kubeconfig, "get", kind, "-n", namespace, "-o", "json"))["items"] + for item in items: + replicas[f"{kind}/{item['metadata']['name']}"] = item["spec"].get("replicas") + return {"restarts": restarts, "pod_uids": pod_uids, "replicas": replicas} + + +# --------------------------------------------------------------------------- +# Bundle helpers +# --------------------------------------------------------------------------- +def _expected_log_files(pods: list[dict]) -> set[str]: + """Derive expected per-replica log filenames (data-model.md naming rules).""" + expected: set[str] = set() + for pod in pods: + name = pod["metadata"]["name"] + containers = [status["name"] for status in pod["status"].get("containerStatuses", [])] + if len(containers) > 1: + expected.update(f"{name}_{container}.log" for container in containers) + else: + expected.add(f"{name}.log") + return expected + + +def _previous_log_name(pod: dict, container: str) -> str: + """Derive the .previous.log filename for one restarted container.""" + name = pod["metadata"]["name"] + containers = pod["status"].get("containerStatuses", []) + base = f"{name}_{container}" if len(containers) > 1 else name + return f"{base}.previous.log" + + +# --------------------------------------------------------------------------- +# Test +# --------------------------------------------------------------------------- +@pytest.mark.e2e +@pytest.mark.k8s +async def test_collect_k8s_bundle(infrahub_k8s, collect_binary, tmp_path): + """K8s: full collect — archive, manifest, per-replica + previous logs, SC-003.""" + namespace = infrahub_k8s["namespace"] + kubeconfig = infrahub_k8s["kubeconfig_path"] + output_dir = tmp_path / "bundles" + env = {"KUBECONFIG": kubeconfig} + + deployment = _find_task_worker_deployment(kubeconfig, namespace) + original_replicas = int( + _kubectl(kubeconfig, "get", "deployment", deployment, "-n", namespace, "-o", "jsonpath={.spec.replicas}") + ) + + try: + # 1. Scale task-worker to >= 2 replicas so per-replica logs are observable + if original_replicas < 2: + _scale_deployment(kubeconfig, namespace, deployment, 2) + _wait_pods_ready(kubeconfig, namespace, "task-worker") + + worker_pods = _service_pods(kubeconfig, namespace, "task-worker") + assert len(worker_pods) >= 2, f"Expected >= 2 task-worker pods, found {len(worker_pods)}" + + # 2. Induce an in-place container restart so previous logs exist + restart_pod = worker_pods[0] + restart_container = restart_pod["status"]["containerStatuses"][0]["name"] + _induce_container_restart(kubeconfig, namespace, restart_pod["metadata"]["name"], restart_container) + _wait_pods_ready(kubeconfig, namespace, "task-worker") + + # Refresh pod view after the induced restart (same pods, bumped restartCount) + worker_pods = _service_pods(kubeconfig, namespace, "task-worker") + restart_pod = next(pod for pod in worker_pods if pod["metadata"]["name"] == restart_pod["metadata"]["name"]) + server_pods = _service_pods(kubeconfig, namespace, "infrahub-server") + assert server_pods, "No infrahub-server pods found" + + # 3. SC-003 baseline: restart counts, pod identities, workload replicas + before = _cluster_state(kubeconfig, namespace) + + # 4. Run the collect + run_collect( + collect_binary, + ["--k8s-namespace", namespace, "--output-dir", str(output_dir), "create"], + env=env, + ) + + # 5. SC-003: the run caused zero restarts, pod replacements, or scale events + after = _cluster_state(kubeconfig, namespace) + assert after["restarts"] == before["restarts"], "Collect run changed container restart counts" + assert after["pod_uids"] == before["pod_uids"], "Collect run replaced pods" + assert after["replicas"] == before["replicas"], "Collect run changed workload replica counts" + + # 6. Archive integrity: valid tar.gz, everything under bundle/ + bundle_path = find_bundle(output_dir) + file_names, manifest = read_bundle_archive(bundle_path) + + # 7. Manifest conforms to contracts/manifest.schema.json + validate_manifest_schema(manifest) + assert manifest["environment"] == "kubernetes" + assert f"support_bundle_{manifest['collect_id']}.tar.gz" == bundle_path.name + + # SC-005: one manifest entry per planned collector, no duplicates + statuses = assert_collector_outcomes(manifest) + assert statuses["logs/infrahub-server"] == "success" + assert statuses["logs/task-worker"] == "success" + + # 8. Layout per contracts/bundle-layout.md + assert_bundle_layout(file_names, statuses) + + # 9. One log file per replica for the multi-replica service (FR-002) + worker_logs = { + name.removeprefix("bundle/logs/task-worker/") + for name in file_names + if name.startswith("bundle/logs/task-worker/") + } + expected_worker_logs = _expected_log_files(worker_pods) + assert expected_worker_logs <= worker_logs, ( + f"Missing per-replica task-worker logs: {expected_worker_logs - worker_logs}" + ) + + server_logs = { + name.removeprefix("bundle/logs/infrahub-server/") + for name in file_names + if name.startswith("bundle/logs/infrahub-server/") + } + assert _expected_log_files(server_pods) <= server_logs, "Missing per-replica infrahub-server logs" + + # 10. Previous-container log for the induced restart (FR-003) + previous_log = _previous_log_name(restart_pod, restart_container) + assert previous_log in worker_logs, ( + f"Expected {previous_log} for the restarted container, found: {sorted(worker_logs)}" + ) + finally: + # Leave the shared session cluster as we found it + _scale_deployment(kubeconfig, namespace, deployment, original_replicas) diff --git a/tests/helpers/bundle.py b/tests/helpers/bundle.py new file mode 100644 index 0000000..c5a5390 --- /dev/null +++ b/tests/helpers/bundle.py @@ -0,0 +1,172 @@ +"""Shared assertions for troubleshooting-bundle e2e tests. + +Used by both test_docker_collect.py and test_k8s_collect.py so the +cross-environment parity promised by spec 003-collect-tool (FR-006 / SC-004) +is asserted with the same code on both sides. +""" + +import json +import re +import tarfile +from datetime import datetime +from pathlib import Path + +# Full planned collector set (SC-005: the manifest accounts for every planned +# collector, present or not) — logs per canonical service plus the parity +# diagnostics and metrics collectors. +LOG_SERVICES = [ + "infrahub-server", + "task-worker", + "database", + "message-queue", + "cache", + "task-manager", + "task-manager-db", + "task-manager-background-svc", +] +EXPECTED_COLLECTORS = {f"logs/{service}" for service in LOG_SERVICES} | { + "database-logs", + "message-queue-status", + "cache-status", + "task-worker-state", + "task-manager-state", + "server-info", + "metrics", + "benchmark", + "backup", +} + +# Opt-in extras: always present in the manifest, skipped ("not requested") +# unless their flag is set. +OPT_IN_COLLECTORS = {"backup", "benchmark"} + +# Bundle directory owned by each non-log collector (contracts/bundle-layout.md). +COLLECTOR_DIRS = { + "database-logs": "database", + "message-queue-status": "message-queue", + "cache-status": "cache", + "task-worker-state": "task-worker", + "task-manager-state": "task-manager", + "server-info": "server", + "metrics": "metrics", +} + +ALLOWED_TOP_LEVEL = {"bundle_information.json", "logs", "benchmark"} | set(COLLECTOR_DIRS.values()) + +COLLECT_ID_RE = re.compile(r"^[0-9]{8}_[0-9]{6}$") + + +def find_bundle(output_dir: Path) -> Path: + bundles = sorted(output_dir.glob("support_bundle_*.tar.gz")) + assert bundles, f"No support bundle found in {output_dir}" + return bundles[-1] + + +def read_bundle_archive(bundle_path: Path) -> tuple[set[str], dict]: + """Validate archive integrity and return (file member names, manifest). + + Asserts every member lives under bundle/ and fully decompresses each file + so a truncated or corrupt archive fails here. + """ + with tarfile.open(bundle_path, "r:gz") as tar: + members = tar.getmembers() + for member in members: + assert member.name == "bundle" or member.name.startswith("bundle/"), ( + f"Archive member outside bundle/: {member.name}" + ) + if member.isfile(): + extracted = tar.extractfile(member) + assert extracted is not None + extracted.read() # full decompression validates integrity + file_names = {member.name for member in members if member.isfile()} + manifest_file = tar.extractfile("bundle/bundle_information.json") + assert manifest_file is not None, "bundle_information.json missing from archive" + manifest = json.loads(manifest_file.read()) + return file_names, manifest + + +def read_bundle_member(bundle_path: Path, member: str) -> str: + """Return the text content of one file inside the bundle archive.""" + with tarfile.open(bundle_path, "r:gz") as tar: + extracted = tar.extractfile(member) + assert extracted is not None, f"{member} missing from archive" + return extracted.read().decode() + + +def validate_manifest_schema(manifest: dict) -> None: + """Assert manifest conformance with contracts/manifest.schema.json. + + jsonschema is not a test dependency; the schema's constraints are asserted + manually (required fields, types, collect_id pattern, created_at format, + environment/status enums, reason required on failed/skipped). + """ + required = { + "manifest_version": int, + "collect_id": str, + "created_at": str, + "tool_version": str, + "infrahub_version": str, + "environment": str, + "log_lines": int, + "collectors": list, + } + for field, expected_type in required.items(): + assert field in manifest, f"Manifest missing required field {field!r}" + value = manifest[field] + assert isinstance(value, expected_type) and not isinstance(value, bool), ( + f"Manifest field {field!r} must be {expected_type.__name__}, got {type(value).__name__}" + ) + + assert COLLECT_ID_RE.match(manifest["collect_id"]), f"collect_id {manifest['collect_id']!r} violates pattern" + datetime.fromisoformat(manifest["created_at"]) # RFC3339 date-time + assert manifest["environment"] in ("docker", "kubernetes"), f"Invalid environment {manifest['environment']!r}" + assert manifest["log_lines"] >= 1, "log_lines must be >= 1" + assert manifest["collectors"], "collectors must have at least one entry" + + for entry in manifest["collectors"]: + assert isinstance(entry, dict), f"Collector entry must be an object: {entry!r}" + assert isinstance(entry.get("name"), str) and entry["name"], f"Collector entry missing name: {entry!r}" + assert entry.get("status") in ("success", "failed", "skipped"), ( + f"Collector {entry.get('name')!r} has invalid status {entry.get('status')!r}" + ) + if entry["status"] in ("failed", "skipped"): + assert isinstance(entry.get("reason"), str) and entry["reason"], ( + f"Collector {entry['name']!r} is {entry['status']} but has no reason" + ) + if "artifact" in entry: + assert isinstance(entry["artifact"], str) and entry["artifact"], ( + f"Collector {entry['name']!r} has a non-string or empty artifact: {entry['artifact']!r}" + ) + assert entry["status"] == "success", ( + f"Collector {entry['name']!r} carries an artifact but is {entry['status']}" + ) + + +def assert_collector_outcomes(manifest: dict) -> dict[str, str]: + """SC-005: one manifest entry per planned collector, no duplicates. + + Returns the {collector name: status} mapping for further assertions. + """ + names = [entry["name"] for entry in manifest["collectors"]] + assert len(names) == len(set(names)), f"Duplicate collector entries: {names}" + assert set(names) == EXPECTED_COLLECTORS, ( + f"Manifest collector set mismatch: missing {EXPECTED_COLLECTORS - set(names)}, " + f"unexpected {set(names) - EXPECTED_COLLECTORS}" + ) + return {entry["name"]: entry["status"] for entry in manifest["collectors"]} + + +def assert_bundle_layout(file_names: set[str], statuses: dict[str, str]) -> None: + """Layout per contracts/bundle-layout.md: successful collectors own files.""" + top_level = {name.split("/")[1] for name in file_names if name.count("/") >= 1} + assert top_level <= ALLOWED_TOP_LEVEL, f"Unexpected top-level bundle entries: {top_level - ALLOWED_TOP_LEVEL}" + for collector, directory in COLLECTOR_DIRS.items(): + if statuses[collector] == "success": + assert any(name.startswith(f"bundle/{directory}/") for name in file_names), ( + f"Collector {collector} succeeded but bundle/{directory}/ is empty" + ) + for service in LOG_SERVICES: + if statuses[f"logs/{service}"] == "success": + assert any(name.startswith(f"bundle/logs/{service}/") for name in file_names), ( + f"logs/{service} succeeded but bundle/logs/{service}/ is empty" + ) diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 18fe007..321574e 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -92,6 +92,22 @@ def run_backup( return result +def run_collect( + binary: str, + extra_args: list[str], + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess: + """Run infrahub-collect with the given arguments.""" + cmd = [binary] + extra_args + run_env = {**os.environ, **(env or {})} + result = subprocess.run(cmd, capture_output=True, text=True, env=run_env) + if result.returncode != 0: + raise RuntimeError( + f"Collect failed (exit {result.returncode}):\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) + return result + + def run_restore( binary: str, extra_args: list[str],