diff --git a/.env.sample b/.env.sample index 59074d1..17a893d 100644 --- a/.env.sample +++ b/.env.sample @@ -5,3 +5,21 @@ APP_HOST=0.0.0.0 APP_PORT=5000 DATABASE_URL=stackdog.db RUST_BACKTRACE=full + +# Log Sniff Configuration +#STACKDOG_LOG_SOURCES=/var/log/syslog,/var/log/auth.log +#STACKDOG_SNIFF_INTERVAL=30 +#STACKDOG_SNIFF_OUTPUT_DIR=./stackdog-logs/ + +# AI Provider Configuration +# Supports OpenAI, Ollama (http://localhost:11434/v1), or any OpenAI-compatible API +#STACKDOG_AI_PROVIDER=openai +#STACKDOG_AI_API_URL=http://localhost:11434/v1 +#STACKDOG_AI_API_KEY= +#STACKDOG_AI_MODEL=llama3 + +# Notification Channels +# Slack: create an incoming webhook at https://api.slack.com/messaging/webhooks +#STACKDOG_SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T.../B.../xxxxx +# Generic webhook endpoint for alert notifications +#STACKDOG_WEBHOOK_URL=https://example.com/webhook diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..2b679aa --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,113 @@ +# Stackdog Security — Copilot Instructions + +## What This Project Is + +Stackdog is a Rust-based security platform for Docker containers and Linux servers. It collects events via eBPF syscall monitoring, runs them through a rule/signature engine and optional ML anomaly detection, manages firewall responses (nftables/iptables + container quarantine), and exposes a REST + WebSocket API consumed by a React/TypeScript dashboard. + +## Workspace Structure + +This is a Cargo workspace with two crates: +- `.` — Main crate (`stackdog`): HTTP server, all security logic +- `ebpf/` — Separate crate (`stackdog-ebpf`): eBPF programs compiled for the kernel (uses `aya-ebpf`) + +## Build, Test, and Lint Commands + +```bash +# Build +cargo build +cargo build --release + +# Tests +cargo test --lib # Unit tests only (in-source) +cargo test --all # All tests including integration +cargo test --lib -- events:: # Run tests for a specific module +cargo test --lib -- rules::scorer # Run a single test by name prefix + +# Code quality +cargo fmt --all +cargo clippy --all +cargo audit # Dependency vulnerability scan + +# Benchmarks +cargo bench + +# Frontend (in web/) +npm test +npm run lint +npm run build +``` + +## Environment Setup + +Requires a `.env` file (copy `.env.sample`). Key variables: +``` +APP_HOST=0.0.0.0 +APP_PORT=5000 +DATABASE_URL=stackdog.db +RUST_BACKTRACE=full +``` + +System dependencies (Linux): `libsqlite3-dev libssl-dev clang llvm pkg-config` + +## Architecture + +``` +Collectors (Linux only) Rule Engine Response + eBPF syscall events → Signatures → nftables/iptables + Docker daemon events → Threat scoring → Container quarantine + Network events → ML anomaly det. → Alerting + + REST + WebSocket API + React/TypeScript UI +``` + +**Key src/ modules:** + +| Module | Purpose | +|---|---| +| `events/` | Core event types: `SyscallEvent`, `SecurityEvent`, `NetworkEvent`, `ContainerEvent` | +| `rules/` | Rule engine, signature database, threat scorer | +| `alerting/` | `AlertManager`, notification channels (Slack/email/webhook) | +| `collectors/` | eBPF loader, Docker daemon events, network collector (Linux only) | +| `firewall/` | nftables management, iptables fallback, `QuarantineManager` (Linux only) | +| `ml/` | Candle-based anomaly detection (optional `ml` feature) | +| `correlator/` | Event correlation engine | +| `baselines/` | Baseline learning for anomaly detection | +| `database/` | SQLite connection pool (`r2d2` + raw `rusqlite`), repositories | +| `api/` | actix-web REST endpoints + WebSocket | +| `response/` | Automated response action pipeline | + +## Key Conventions + +### Platform-Gating +Linux-only modules (`collectors`, `firewall`) and deps (aya, netlink) are gated: +```rust +#[cfg(target_os = "linux")] +pub mod firewall; +``` +The `ebpf` and `ml` features are opt-in and must be enabled explicitly: +```bash +cargo build --features ebpf +cargo build --features ml +``` + +### Error Handling +- Use `anyhow::{Result, Context}` for application/binary code +- Use `thiserror` for library error types +- Never use `.unwrap()` in production code; use `?` with `.context("...")` + +### Database +The project uses raw `rusqlite` with `r2d2` connection pooling. `DbPool` is `r2d2::Pool`. Tables are created with `CREATE TABLE IF NOT EXISTS` in `database::connection::init_database`. Repositories are in `src/database/repositories/` and receive a `&DbPool`. + +### API Routes +Each API sub-module exports a `configure_routes(cfg: &mut web::ServiceConfig)` function. All routes are composed in `api::configure_all_routes`, which is the single call site in `main.rs`. + +### Test Location +- **Unit tests**: `#[cfg(test)] mod tests { ... }` inside source files +- **Integration tests**: `tests/` directory at workspace root + +### eBPF Programs +The `ebpf/` crate is compiled separately for the Linux kernel. User-space loading is handled by `src/collectors/ebpf/` using the `aya` library. Kernel-side programs use `aya-ebpf`. + +### Async Runtime +The main binary uses `#[actix_rt::main]`. Library code uses `tokio`. Avoid mixing runtimes. diff --git a/.github/workflows/codacy-analysis.yml b/.github/workflows/codacy-analysis.yml index 46ec09a..93c44dd 100644 --- a/.github/workflows/codacy-analysis.yml +++ b/.github/workflows/codacy-analysis.yml @@ -21,7 +21,7 @@ jobs: steps: # Checkout the repository to the GitHub Actions runner - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis - name: Run Codacy Analysis CLI @@ -41,6 +41,6 @@ jobs: # Upload the SARIF file generated in the previous step - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@v1 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: results.sarif diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d3a0eac..52cc06a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,10 +12,11 @@ on: jobs: cicd-linux-docker: name: Cargo and npm build - runs-on: ubuntu-latest + #runs-on: ubuntu-latest + runs-on: [self-hosted, linux] steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install stable toolchain uses: actions-rs/toolchain@v1 @@ -26,7 +27,7 @@ jobs: components: rustfmt, clippy - name: Cache cargo registry - uses: actions/cache@v2.1.6 + uses: actions/cache@v4 with: path: ~/.cargo/registry key: docker-registry-${{ hashFiles('**/Cargo.lock') }} @@ -35,7 +36,7 @@ jobs: docker- - name: Cache cargo index - uses: actions/cache@v2.1.6 + uses: actions/cache@v4 with: path: ~/.cargo/git key: docker-index-${{ hashFiles('**/Cargo.lock') }} @@ -48,7 +49,7 @@ jobs: head -c16 /dev/urandom > src/secret.key - name: Cache cargo build - uses: actions/cache@v2.1.6 + uses: actions/cache@v4 with: path: target key: docker-build-${{ hashFiles('**/Cargo.lock') }} @@ -101,7 +102,7 @@ jobs: # npm test - name: Archive production artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: dist-without-markdown path: | @@ -109,7 +110,7 @@ jobs: !web/dist/**/*.md # - name: Archive code coverage results -# uses: actions/upload-artifact@v2 +# uses: actions/upload-artifact@v4 # with: # name: code-coverage-report # path: output/test/code-coverage.html @@ -128,18 +129,19 @@ jobs: cd .. - name: Upload app archive for Docker job - uses: actions/upload-artifact@v2.2.2 + uses: actions/upload-artifact@v4 with: name: artifact-linux-docker path: app.tar.gz cicd-docker: name: CICD Docker - runs-on: ubuntu-latest + #runs-on: ubuntu-latest + runs-on: [self-hosted, linux] needs: cicd-linux-docker steps: - name: Download app archive - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: artifact-linux-docker @@ -149,12 +151,14 @@ jobs: - name: Display structure of downloaded files run: ls -R - - name: Docker build and publish - uses: docker/build-push-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - repository: trydirect/stackdog - add_git_labels: true - tag_with_ref: true - #no-cache: true + + - name: Docker build and publish + uses: docker/build-push-action@v6 + with: + push: true + tags: trydirect/stackdog:latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f15bf4c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,77 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + artifact: stackdog-linux-x86_64 + - target: aarch64-unknown-linux-gnu + artifact: stackdog-linux-aarch64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross + run: cargo install cross --git https://github.com/cross-rs/cross + + - name: Build release binary + run: cross build --release --target ${{ matrix.target }} + + - name: Package + run: | + mkdir -p dist + cp target/${{ matrix.target }}/release/stackdog dist/stackdog + cd dist + tar czf ${{ matrix.artifact }}.tar.gz stackdog + sha256sum ${{ matrix.artifact }}.tar.gz > ${{ matrix.artifact }}.tar.gz.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: | + dist/${{ matrix.artifact }}.tar.gz + dist/${{ matrix.artifact }}.tar.gz.sha256 + + release: + name: Create GitHub Release + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Create release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: | + artifacts/*.tar.gz + artifacts/*.sha256 diff --git a/.gitignore b/.gitignore index 49c5b70..89b9d61 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ Cargo.lock ======= *.db >>>>>>> testing +docs/tasks/ diff --git a/.qwen/PROJECT_MEMORY.md b/.qwen/PROJECT_MEMORY.md new file mode 100644 index 0000000..61d707c --- /dev/null +++ b/.qwen/PROJECT_MEMORY.md @@ -0,0 +1,277 @@ +# Stackdog Security - Project Memory + +## Project Identity + +**Name:** Stackdog Security +**Version:** 0.1.0 (Security-focused rewrite) +**Type:** Container and Linux Server Security Platform +**License:** MIT + +## Core Mission + +> Provide real-time security monitoring, AI-powered threat detection, and automated response for Docker containers and Linux servers using Rust and eBPF technologies. + +## Key Decisions + +### Architecture Decisions + +| ID | Decision | Rationale | Date | +|----|----------|-----------|------| +| **ARCH-001** | Use eBPF for syscall monitoring | Minimal overhead (<5% CPU), kernel-level visibility, safe (sandboxed) | 2026-03-13 | +| **ARCH-002** | Use Candle for ML instead of Python | Native Rust, no Python dependencies, fast inference, maintained by HuggingFace | 2026-03-13 | +| **ARCH-003** | Use nftables over iptables | Modern, faster, better batch support, iptables as fallback | 2026-03-13 | +| **ARCH-004** | TDD development methodology | Better code quality, maintainability, regression prevention | 2026-03-13 | +| **ARCH-005** | Functional programming principles | Immutability, fewer bugs, easier reasoning about code | 2026-03-13 | + +### Technology Choices + +| Component | Technology | Alternatives Considered | +|-----------|-----------|------------------------| +| **eBPF Framework** | aya-rs | libbpf (C), bcc (Python) | +| **ML Framework** | Candle (HuggingFace) | PyTorch (Python), ONNX Runtime, linfa | +| **Web Framework** | Actix-web 4.x | Axum, Rocket | +| **Database** | SQLite + rusqlite + r2d2 | PostgreSQL, Redis | +| **Firewall** | nftables (netlink) | iptables, firewalld | + +## Project Structure + +``` +stackdog/ +├── src/ +│ ├── collectors/ # Event collection (eBPF, Docker, etc.) +│ ├── events/ # Event types and structures +│ ├── ml/ # ML engine (Candle-based) +│ ├── firewall/ # Firewall management (nftables/iptables) +│ ├── response/ # Automated response actions +│ ├── correlator/ # Event correlation +│ ├── alerting/ # Alert system +│ ├── api/ # REST API + WebSocket +│ ├── config/ # Configuration +│ ├── models/ # Data models +│ ├── database/ # Database operations +│ └── utils/ # Utilities +├── ebpf/ # eBPF programs (separate crate) +├── web/ # React/TypeScript frontend +├── tests/ # Integration tests +├── benches/ # Performance benchmarks +└── models/ # Pre-trained ML models +``` + +## Development Principles + +### Clean Code (Robert C. Martin) + +1. **DRY** - Don't Repeat Yourself +2. **SRP** - Single Responsibility Principle +3. **OCP** - Open/Closed Principle +4. **DIP** - Dependency Inversion Principle +5. **Functional First** - Immutability, From/Into traits, builder pattern + +### TDD Workflow + +``` +Red → Green → Refactor +``` + +1. Write failing test +2. Run test (verify failure) +3. Implement minimal code to pass +4. Run test (verify pass) +5. Refactor (maintain passing tests) + +### Code Review Checklist + +- [ ] Tests written first (TDD) +- [ ] All tests pass +- [ ] Code formatted (`cargo fmt`) +- [ ] No clippy warnings +- [ ] DRY principle followed +- [ ] Functions < 50 lines +- [ ] Error handling comprehensive +- [ ] Documentation for public APIs + +## Key APIs and Interfaces + +### Event Types + +```rust +// Core security event +pub enum SecurityEvent { + Syscall(SyscallEvent), + Network(NetworkEvent), + Container(ContainerEvent), + Alert(AlertEvent), +} + +// Syscall event from eBPF +pub struct SyscallEvent { + pub pid: u32, + pub uid: u32, + pub syscall_type: SyscallType, + pub timestamp: DateTime, + pub container_id: Option, +} +``` + +### ML Interface + +```rust +// Feature vector for ML +pub struct SecurityFeatures { + pub syscall_rate: f64, + pub network_rate: f64, + pub unique_processes: u32, + pub privileged_calls: u32, + // ... +} + +// Threat score output +pub enum ThreatScore { + Normal, + Low, + Medium, + High, + Critical, +} +``` + +### Firewall Interface + +```rust +pub trait FirewallBackend { + fn add_rule(&self, rule: &Rule) -> Result<()>; + fn remove_rule(&self, rule: &Rule) -> Result<()>; + fn batch_update(&self, rules: &[Rule]) -> Result<()>; + fn block_container(&self, container_id: &str) -> Result<()>; + fn quarantine_container(&self, container_id: &str) -> Result<()>; +} +``` + +## Configuration + +### Environment Variables + +```bash +APP_HOST=0.0.0.0 +APP_PORT=5000 +DATABASE_URL=stackdog.db +RUST_LOG=info +RUST_BACKTRACE=full + +# Security-specific +EBPF_ENABLED=true +FIREWALL_BACKEND=nftables # or iptables +ML_ENABLED=true +ML_MODEL_PATH=models/ +ALERT_THRESHOLD=0.75 +``` + +### Cargo Features + +```toml +[features] +default = ["nftables", "ml"] +nftables = ["netlink-packet-route"] +iptables = ["iptables"] +ml = ["candle-core", "candle-nn"] +ebpf = ["aya"] +``` + +## Testing Strategy + +### Test Categories + +| Category | Location | Command | Coverage Target | +|----------|----------|---------|-----------------| +| Unit | `src/**/*.rs` | `cargo test` | 80%+ | +| Integration | `tests/integration/` | `cargo test --test integration` | Critical paths | +| E2E | `tests/e2e/` | `cargo test --test e2e` | Key workflows | +| Benchmark | `benches/` | `cargo bench` | Performance targets | + +### Performance Targets + +| Metric | Target | +|--------|--------| +| Event throughput | 100K events/sec | +| ML inference latency | <10ms | +| Firewall update | <1ms per rule | +| Memory usage | <256MB baseline | +| CPU overhead | <5% | + +## Dependencies + +### Core + +- `actix-web` - Web framework +- `aya` - eBPF framework +- `candle-core`, `candle-nn` - ML framework +- `bollard` - Docker API +- `rusqlite` - SQLite driver +- `r2d2` - Connection pool +- `netlink-packet-route` - nftables +- `tokio` - Async runtime + +### Development + +- `mockall` - Mocking for tests +- `criterion` - Benchmarking +- `cargo-audit` - Security audit +- `cargo-deny` - Dependency linting + +## Milestones + +| Version | Target | Features | +|---------|--------|----------| +| v0.1.0 | Week 4 | eBPF collectors, basic rules | +| v0.2.0 | Week 6 | Firewall integration | +| v0.3.0 | Week 10 | ML anomaly detection | +| v0.4.0 | Week 12 | Alerting system | +| v0.5.0 | Week 16 | Web dashboard | +| v1.0.0 | Week 18 | Production release | + +## Risks and Mitigations + +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| eBPF kernel compatibility | High | Medium | Fallback to auditd | +| ML model accuracy | High | Medium | Start with rule-based, iterate | +| Performance overhead | High | Low | Benchmark early, optimize | +| False positives | Medium | High | Tunable thresholds, learning period | + +## Open Questions + +1. **Model Training:** How to collect training data for ML models? + - Decision: Start with synthetic data, then real-world collection + +2. **Multi-node Support:** Single node first, cluster later? + - Decision: Single node for v1.0, cluster in v2.0 + +3. **Kubernetes Support:** Include in scope? + - Decision: Out of scope for v1.0, backlog for v2.0 + +## Resources + +### Documentation + +- [DEVELOPMENT.md](DEVELOPMENT.md) - Full development plan +- [TODO.md](TODO.md) - Task tracking +- [BUGS.md](BUGS.md) - Bug tracking +- [CONTRIBUTING.md](CONTRIBUTING.md) - Contribution guidelines + +### External + +- [Rust Book](https://doc.rust-lang.org/book/) +- [Candle Docs](https://docs.rs/candle-core) +- [aya-rs Docs](https://aya-rs.dev/) +- [eBPF Documentation](https://ebpf.io/) + +## Contact + +- **Project Lead:** Vasili Pascal +- **Email:** info@try.direct +- **Twitter:** [@VasiliiPascal](https://twitter.com/VasiliiPascal) +- **Gitter:** [stackdog/community](https://gitter.im/stackdog/community) + +--- + +*Last updated: 2026-03-13* diff --git a/CHANGELOG.md b/CHANGELOG.md index cd47a13..a6b9bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,71 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +#### Log Sniffing & Analysis (`stackdog sniff`) +- **CLI Subcommands** — Multi-mode binary with `stackdog serve` and `stackdog sniff` + - `--once` flag for single-pass mode + - `--consume` flag to archive logs (zstd) and purge originals + - `--sources` to add custom log paths + - `--ai-provider` to select AI backend (openai/candle) + - `--interval` for polling frequency + - `--output` for archive destination + +- **Log Source Discovery** — Automatic and manual log source management + - System logs (`/var/log/syslog`, `messages`, `auth.log`, etc.) + - Docker container logs via bollard API + - Custom file paths (CLI, env var, or REST API) + - Incremental read position tracking (byte offset persisted in DB) + +- **Log Readers** — Trait-based reader abstraction + - `FileLogReader` with byte-offset tracking and log rotation detection + - `DockerLogReader` using bollard streaming API + - `JournaldReader` (Linux-gated) for systemd journal + +- **AI-Powered Analysis** — Dual-backend log summarization + - `OpenAiAnalyzer` — works with any OpenAI-compatible API (OpenAI, Ollama, vLLM) + - `PatternAnalyzer` — local fallback with error/warning counting and spike detection + - Structured `LogSummary` with anomaly detection (`LogAnomaly`, severity levels) + +- **Log Consumer** — Archive and purge pipeline + - FNV hash-based deduplication + - zstd compression (level 3) for archived logs + - File truncation and Docker log purge + - `ConsumeResult` tracking (entries archived, duplicates skipped, bytes freed) + +- **Reporter** — Bridges log analysis to existing alert system + - Converts `LogAnomaly` → `Alert` using `AlertManager` infrastructure + - Routes notifications via `route_by_severity()` to configured channels + - Persists `LogSummary` records to database + +- **REST API Endpoints** + - `GET /api/logs/sources` — list discovered log sources + - `POST /api/logs/sources` — manually add a custom source + - `GET /api/logs/sources/{path}` — get source details + - `DELETE /api/logs/sources/{path}` — remove a source + - `GET /api/logs/summaries` — list AI-generated summaries (filterable by source) + +- **Database Tables** — `log_sources` and `log_summaries` with indexes + +#### Dependencies +- `clap = "4"` (derive) — CLI argument parsing +- `async-trait = "0.1"` — async trait support +- `reqwest = "0.12"` (json) — HTTP client for AI APIs +- `zstd = "0.13"` — log compression +- `futures-util = "0.3"` — Docker log streaming + +### Changed + +- Refactored `main.rs` to dispatch `serve`/`sniff` subcommands via clap +- Added `events`, `rules`, `alerting`, `models` modules to binary crate +- Updated `.env.sample` with `STACKDOG_LOG_SOURCES`, `STACKDOG_AI_*` config vars + +### Testing + +- **80+ new tests** covering all sniff modules (TDD) + - Config: 12, Discovery: 14, Readers: 10, Analyzer: 16, Consumer: 13, Reporter: 5, Orchestrator: 3, API: 7 + ### Planned - Web dashboard (React/TypeScript) diff --git a/Cargo.toml b/Cargo.toml index fd6a1e1..cf82f97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ tracing-subscriber = "0.3" dotenv = "0.15" anyhow = "1" thiserror = "1" +clap = { version = "4", features = ["derive"] } # Async runtime tokio = { version = "1", features = ["full"] } @@ -37,6 +38,7 @@ actix-web = "4" actix-cors = "0.6" actix-web-actors = "4" actix = "0.13" +async-trait = "0.1" # Database rusqlite = { version = "0.32", features = ["bundled"] } @@ -45,6 +47,15 @@ r2d2 = "0.8" # Docker bollard = "0.16" +# HTTP client (for LLM API) +reqwest = { version = "0.12", features = ["json", "blocking"] } + +# Compression +zstd = "0.13" + +# Stream utilities +futures-util = "0.3" + # eBPF (Linux only) [target.'cfg(target_os = "linux")'.dependencies] aya = "0.12" @@ -66,6 +77,7 @@ ebpf = [] [dev-dependencies] # Testing tokio-test = "0.4" +tempfile = "3" # Benchmarking criterion = { version = "0.5", features = ["html_reports"] } diff --git a/QWEN.md b/QWEN.md new file mode 100644 index 0000000..9ce8ee0 --- /dev/null +++ b/QWEN.md @@ -0,0 +1,311 @@ +# Stackdog Security - Project Context + +## Project Overview + +**Stackdog Security** is a Rust-based security platform for Docker containers and Linux servers. It provides real-time threat detection, AI-powered anomaly detection using Candle (HuggingFace's Rust ML framework), and automated response through firewall management (nftables/iptables). + +### Core Capabilities + +1. **Real-time Monitoring** — System events via eBPF (aya-rs), network traffic, and container activity +2. **AI/ML Detection** — Anomaly detection using Candle (native Rust, no Python) +3. **Automated Response** — Fast nftables/iptables management and container quarantine +4. **Security Dashboard** — Web UI for threat visualization and management + +### Key Technologies + +| Component | Technology | Rationale | +|-----------|-----------|-----------| +| **Core Language** | Rust 2021 | Performance, safety, concurrency | +| **ML Framework** | Candle (HuggingFace) | Native Rust, fast inference, no Python dependencies | +| **eBPF** | aya-rs | Pure Rust eBPF framework, minimal overhead | +| **Firewall** | nftables (netlink) | Modern, faster than iptables | +| **Web Framework** | Actix-web 4.x | High performance | +| **Database** | SQLite + rusqlite + r2d2 | Embedded, low overhead | + +--- + +## Architecture + +``` +stackdog/ +├── src/ +│ ├── collectors/ # Event collection (eBPF, Docker, network) +│ ├── events/ # Event types (SyscallEvent, SecurityEvent) +│ ├── ml/ # ML engine (Candle-based anomaly detection) +│ ├── firewall/ # Firewall management (nftables/iptables) +│ ├── response/ # Automated response actions +│ ├── correlator/ # Event correlation engine +│ ├── alerting/ # Alert system and notifications +│ ├── api/ # REST API + WebSocket +│ ├── config/ # Configuration +│ ├── models/ # Data models +│ ├── database/ # Database operations +│ └── utils/ # Utilities +├── ebpf/ # eBPF programs (separate crate) +├── web/ # React/TypeScript frontend +├── tests/ # Integration and E2E tests +├── benches/ # Performance benchmarks +└── models/ # Pre-trained ML models +``` + +--- + +## Development Status + +**Current Phase:** Phase 1 - Foundation & eBPF Collectors (Weeks 1-4) + +**Active Tasks:** See [TODO.md](TODO.md) + +**Development Plan:** See [DEVELOPMENT.md](DEVELOPMENT.md) + +--- + +## Building and Running + +### Prerequisites + +- Rust 1.75+ (edition 2021) +- SQLite3 + libsqlite3-dev +- Clang + LLVM (for eBPF) +- Kernel 4.19+ (for eBPF with BTF support) +- Docker & Docker Compose (optional) + +### Quick Start + +```bash +# Clone and setup +git clone https://github.com/vsilent/stackdog +cd stackdog + +# Environment setup +cp .env.sample .env + +# Install dependencies (Ubuntu/Debian) +apt-get install libsqlite3-dev libssl-dev clang llvm + +# Build project +cargo build + +# Run tests +cargo test --all + +# Run with debug logging +RUST_LOG=debug cargo run +``` + +### eBPF Development + +```bash +# Install eBPF tools +cargo install cargo-bpf + +# Build eBPF programs +cd ebpf && cargo build --release +``` + +--- + +## Development Commands + +```bash +# Build +cargo build --release + +# Run all tests +cargo test --all + +# Run specific test module +cargo test --test ml::anomaly_detection + +# Linting +cargo clippy --all + +# Formatting +cargo fmt --all -- --check # Check +cargo fmt --all # Fix + +# Performance benchmarks +cargo bench + +# Security audit +cargo audit + +# Watch mode (with cargo-watch) +cargo watch -x test +``` + +--- + +## Testing Strategy (TDD) + +### TDD Workflow + +``` +1. Write failing test +2. Run test (verify failure) +3. Implement minimal code to pass +4. Run test (verify pass) +5. Refactor (maintain passing tests) +``` + +### Test Categories + +| Category | Location | Command | Coverage Target | +|----------|----------|---------|-----------------| +| **Unit Tests** | `src/**/*.rs` | `cargo test` | 80%+ | +| **Integration Tests** | `tests/integration/` | `cargo test --test integration` | Critical paths | +| **E2E Tests** | `tests/e2e/` | `cargo test --test e2e` | Key workflows | +| **Benchmarks** | `benches/` | `cargo bench` | Performance targets | + +### Test Naming Convention + +```rust +#[test] +fn test___() +``` + +Example: +```rust +#[test] +fn test_syscall_event_capture_execve() +#[test] +fn test_isolation_forest_training_valid_data() +#[test] +fn test_container_quarantine_success() +``` + +--- + +## Code Quality Standards + +### Clean Code Principles (Robert C. Martin) + +1. **DRY** - Don't Repeat Yourself +2. **SRP** - Single Responsibility Principle +3. **OCP** - Open/Closed Principle +4. **DIP** - Dependency Inversion Principle +5. **Functional First** - Immutability, `From`/`Into` traits, builder pattern + +### Code Review Checklist + +- [ ] Tests written first (TDD) +- [ ] All tests pass +- [ ] Code formatted (`cargo fmt --all`) +- [ ] No clippy warnings (`cargo clippy --all`) +- [ ] DRY principle followed +- [ ] Functions < 50 lines +- [ ] Error handling comprehensive (`Result` types) +- [ ] Documentation for public APIs + +--- + +## Configuration + +### Environment Variables (`.env`) + +```bash +APP_HOST=0.0.0.0 +APP_PORT=5000 +DATABASE_URL=stackdog.db +RUST_LOG=info +RUST_BACKTRACE=full + +# Security-specific +EBPF_ENABLED=true +FIREWALL_BACKEND=nftables # or iptables +ML_ENABLED=true +ML_MODEL_PATH=models/ +ALERT_THRESHOLD=0.75 +``` + +### Cargo Features + +```toml +[features] +default = ["nftables", "ml"] +nftables = ["netlink-packet-route"] +iptables = ["iptables"] +ml = ["candle-core", "candle-nn"] +ebpf = ["aya"] +``` + +--- + +## Performance Targets + +| Metric | Target | +|--------|--------| +| Event throughput | 100K events/sec | +| ML inference latency | <10ms | +| Firewall update | <1ms per rule | +| Memory usage | <256MB baseline | +| CPU overhead | <5% on monitored host | + +--- + +## Key Files + +| File | Description | +|------|-------------| +| [DEVELOPMENT.md](DEVELOPMENT.md) | Comprehensive development plan with phases | +| [TODO.md](TODO.md) | Task tracking with TDD approach | +| [BUGS.md](BUGS.md) | Bug tracking and reporting | +| [CHANGELOG.md](CHANGELOG.md) | Version history | +| [CONTRIBUTING.md](CONTRIBUTING.md) | Contribution guidelines | +| [ROADMAP.md](ROADMAP.md) | Original roadmap (being updated) | +| `.qwen/PROJECT_MEMORY.md` | Project memory and decisions | + +--- + +## Current Sprint (Phase 1) + +**Goal:** Establish core monitoring infrastructure with eBPF-based syscall collection + +### Active Tasks + +| ID | Task | Status | +|----|------|--------| +| **TASK-001** | Create new project structure for security modules | Pending | +| **TASK-002** | Define security event types | Pending | +| **TASK-003** | Setup aya-rs eBPF integration | Pending | +| **TASK-004** | Implement syscall event capture | Pending | +| **TASK-005** | Create rule engine infrastructure | Pending | + +See [TODO.md](TODO.md) for detailed task descriptions. + +--- + +## Contributing + +1. Pick a task from [TODO.md](TODO.md) or create a new issue +2. Write failing test first (TDD) +3. Implement minimal code to pass +4. Refactor while keeping tests green +5. Submit PR with updated changelog + +### PR Requirements + +- [ ] All tests pass (`cargo test --all`) +- [ ] Code formatted (`cargo fmt --all`) +- [ ] No clippy warnings (`cargo clippy --all`) +- [ ] Changelog updated +- [ ] TDD approach followed + +--- + +## License + +[MIT](LICENSE) + +--- + +## Contact + +- **Project Lead:** Vasili Pascal +- **Email:** info@try.direct +- **Twitter:** [@VasiliiPascal](https://twitter.com/VasiliiPascal) +- **Gitter:** [stackdog/community](https://gitter.im/stackdog/community) + +--- + +*Last updated: 2026-03-13* diff --git a/README.md b/README.md index 41ccfe7..7509523 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,13 @@ ### 🔥 Key Features - **📊 Real-time Monitoring** — eBPF-based syscall monitoring with minimal overhead (<5% CPU) -- **🤖 AI/ML Detection** — Candle-powered anomaly detection (native Rust, no Python) +- **🔍 Log Sniffing** — Discover, read, and AI-summarize logs from containers and system files +- **🤖 AI/ML Detection** — Candle-powered anomaly detection + OpenAI/Ollama log analysis - **🚨 Alert System** — Multi-channel notifications (Slack, email, webhook) - **🔒 Automated Response** — nftables/iptables firewall, container quarantine - **📈 Threat Scoring** — Configurable scoring with time-decay - **🎯 Signature Detection** — 10+ built-in threat signatures +- **📦 Log Archival** — Deduplicate and compress logs with zstd, optionally purge originals --- @@ -42,6 +44,17 @@ ## 🚀 Quick Start +### Install with curl (Linux) + +```bash +curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash +``` + +Pin a specific version: +```bash +curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash -s -- --version v0.2.0 +``` + ### Run as Binary ```bash @@ -49,8 +62,30 @@ git clone https://github.com/vsilent/stackdog cd stackdog -# Build and run +# Start the HTTP server (default) cargo run + +# Or explicitly +cargo run -- serve +``` + +### Log Sniffing + +```bash +# Discover and analyze logs (one-shot) +cargo run -- sniff --once + +# Continuous monitoring with AI analysis +cargo run -- sniff --ai-provider openai + +# Use Ollama (local LLM) +STACKDOG_AI_API_URL=http://localhost:11434/v1 cargo run -- sniff + +# Consume mode: archive to zstd + purge originals +cargo run -- sniff --consume --output ./log-archive + +# Add custom log sources +cargo run -- sniff --sources "/var/log/myapp.log,/opt/service/logs" ``` ### Use as Library @@ -106,6 +141,12 @@ docker-compose logs -f stackdog │ │ • Docker │ │ Detection │ │ • Auto-response │ │ │ │ Events │ │ • Scoring │ │ • Alerting │ │ │ └─────────────┘ └─────────────┘ └─────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────────┐│ +│ │ Log Sniffing ││ +│ │ • Auto-discovery (system logs, Docker, custom paths) ││ +│ │ • AI summarization (OpenAI/Ollama/Candle) ││ +│ │ • zstd compression, dedup, log purge ││ +│ └──────────────────────────────────────────────────────────────┘│ └─────────────────────────────────────────────────────────────────┘ ``` @@ -118,7 +159,8 @@ docker-compose logs -f stackdog | **Alerting** | Alert management & notifications | ✅ Complete | | **Firewall** | nftables/iptables integration | ✅ Complete | | **Collectors** | eBPF syscall monitoring | ✅ Infrastructure | -| **ML** | Candle-based anomaly detection | 🚧 In progress | +| **Log Sniffing** | Log discovery, AI analysis, archival | ✅ Complete | +| **ML** | Candle-based anomaly detection | ⏳ Planned | --- @@ -251,6 +293,44 @@ let action = ResponseAction::new( - Send alerts - Custom commands +### 7. Log Sniffing & AI Analysis + +```bash +# Discover all log sources and analyze with AI +stackdog sniff --once --ai-provider openai + +# Continuous daemon with local Ollama +stackdog sniff --interval 60 --ai-provider openai + +# Consume: archive (zstd) + purge originals to free disk +stackdog sniff --consume --output ./archive + +# Add custom sources alongside auto-discovered ones +stackdog sniff --sources "/app/logs/api.log,/app/logs/worker.log" +``` + +**Capabilities:** +- 🔍 Auto-discovers system logs, Docker container logs, and custom paths +- 🤖 AI summarization via OpenAI, Ollama, or local pattern analysis +- 📦 Deduplicates and compresses logs with zstd +- 🗑️ Optional `--consume` mode: archives then purges originals +- 📊 Incremental reading — tracks byte offsets, never re-reads old entries +- 🚨 Anomaly alerts routed to configured notification channels + +**REST API:** +```bash +# List discovered sources +curl http://localhost:5000/api/logs/sources + +# Add a custom source +curl -X POST http://localhost:5000/api/logs/sources \ + -H 'Content-Type: application/json' \ + -d '{"path": "/var/log/myapp.log", "name": "My App"}' + +# View AI summaries +curl http://localhost:5000/api/logs/summaries?source_id=myapp +``` + --- ## 📦 Installation @@ -297,6 +377,7 @@ cargo test --lib cargo test --lib -- events:: cargo test --lib -- rules:: cargo test --lib -- alerting:: +cargo test --lib -- sniff:: ``` --- @@ -404,11 +485,21 @@ cargo doc --open ``` stackdog/ ├── src/ +│ ├── cli.rs # Clap CLI (serve/sniff subcommands) │ ├── events/ # Event types & validation │ ├── rules/ # Rule engine & signatures │ ├── alerting/ # Alerts & notifications │ ├── firewall/ # nftables/iptables │ ├── collectors/ # eBPF collectors +│ ├── sniff/ # Log sniffing & AI analysis +│ │ ├── config.rs # SniffConfig (env + CLI) +│ │ ├── discovery.rs # Log source auto-discovery +│ │ ├── reader.rs # File/Docker/Journald readers +│ │ ├── analyzer.rs # AI summarization (OpenAI + pattern) +│ │ ├── consumer.rs # zstd compression, dedup, purge +│ │ └── reporter.rs # Alert routing +│ ├── api/ # REST API endpoints +│ ├── database/ # SQLite + repositories │ ├── ml/ # ML infrastructure │ └── config/ # Configuration ├── examples/ # Usage examples @@ -507,11 +598,12 @@ Look for issues labeled: - ✅ Signature detection (TASK-006) - ✅ Alert system (TASK-007) - ✅ Firewall integration (TASK-008) +- ✅ Log sniffing & AI analysis (TASK-009) ### Upcoming Tasks -- ⏳ Web dashboard (TASK-009) - ⏳ ML anomaly detection (TASK-010) +- ⏳ Web dashboard (TASK-011) - ⏳ Kubernetes support (BACKLOG) --- diff --git a/docker-compose.yml b/docker-compose.yml index 381f647..289a2fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,20 +16,15 @@ services: - | echo "Waiting for dependencies..." sleep 5 - echo "Running migrations..." - diesel migration run || echo "Migrations may have already run" echo "Starting Stackdog..." cargo run --bin stackdog ports: - - "5003:5000" + - "${APP_PORT:-8080}:${APP_PORT:-8080}" env_file: - .env environment: - RUST_LOG=debug - RUST_BACKTRACE=full - - APP_HOST=0.0.0.0 - - APP_PORT=5000 - - DATABASE_URL=/app/db/stackdog.db volumes: - db_data:/app/db - ./.env:/app/.env:ro diff --git a/docs/DAY1_PROGRESS.md b/docs/DAY1_PROGRESS.md new file mode 100644 index 0000000..7f5e93a --- /dev/null +++ b/docs/DAY1_PROGRESS.md @@ -0,0 +1,124 @@ +# Day 1 Progress Report - Database Integration + +**Date:** 2026-03-16 +**Status:** ⚠️ Partial Progress + +--- + +## What Was Accomplished + +### ✅ Database Schema Created +- 3 migration files created +- Alerts, threats, containers_cache tables defined +- Indexes for performance + +### ✅ Database Layer Structure +- `src/database/connection.rs` - Connection pool +- `src/database/models/` - Data models +- `src/database/repositories/` - Repository pattern + +### ✅ API Integration Started +- Alerts API updated to use database +- Dependency injection configured +- Main.rs updated with database initialization + +--- + +## Current Blockers + +### Diesel Version Compatibility +The current diesel version (1.4) has API incompatibilities with the migration system. + +**Options:** +1. Upgrade to diesel 2.x (breaking changes) +2. Use raw SQL for everything (more work) +3. Simplify to basic SQL queries (recommended for now) + +--- + +## Recommended Next Steps + +### Option A: Quick Fix (1-2 hours) +Use rusqlite directly instead of diesel: +```toml +[dependencies] +rusqlite = { version = "0.31", features = ["bundled"] } +``` + +Benefits: +- Simpler API +- No migration issues +- Less boilerplate + +### Option B: Full Diesel Upgrade (Half day) +Upgrade to diesel 2.x: +- Update Cargo.toml +- Fix breaking changes +- Update all queries + +### Option C: Hybrid Approach (Recommended) +- Use diesel for connection pooling +- Use raw SQL for queries +- Keep current structure + +--- + +## Files Created Today + +### Migrations +- `migrations/00000000000000_create_alerts/up.sql` +- `migrations/00000000000000_create_alerts/down.sql` +- `migrations/00000000000001_create_threats/*` +- `migrations/00000000000002_create_containers_cache/*` + +### Database Layer +- `src/database/connection.rs` +- `src/database/models/mod.rs` +- `src/database/repositories/alerts.rs` +- `src/database/repositories/mod.rs` +- `src/database/mod.rs` + +### API Updates +- `src/api/alerts.rs` - Updated with DB integration +- `src/main.rs` - Database initialization + +--- + +## Time Spent + +| Task | Time | +|------|------| +| Schema design | 30 min | +| Migration files | 30 min | +| Database layer | 2 hours | +| API integration | 1 hour | +| Debugging diesel | 1 hour | +| **Total** | **5 hours** | + +--- + +## Remaining Work for Day 1 + +### To Complete Database Integration +1. Fix diesel compatibility (30 min) +2. Test database initialization (15 min) +3. Test alert CRUD operations (30 min) +4. Update remaining API endpoints (1 hour) + +**Estimated time:** 2.5 hours + +--- + +## Decision Point + +**Choose one:** + +1. **Continue with diesel** - Fix compatibility issues +2. **Switch to rusqlite** - Simpler, faster implementation +3. **Hybrid approach** - Keep diesel for pooling, raw SQL for queries + +**Recommendation:** Option 3 (Hybrid) - Best balance of speed and maintainability + +--- + +*Report generated: 2026-03-16* diff --git a/docs/DAY2_PLAN.md b/docs/DAY2_PLAN.md new file mode 100644 index 0000000..b9ebeaf --- /dev/null +++ b/docs/DAY2_PLAN.md @@ -0,0 +1,47 @@ +# Day 2: Docker Integration + +**Date:** 2026-03-16 +**Goal:** Connect to Docker API and list real containers + +--- + +## Morning: Docker Client Setup + +### Tasks +- [x] Add bollard dependency +- [ ] Create Docker client wrapper +- [ ] Test Docker connection +- [ ] List containers + +### Files to Create +``` +src/docker/ +├── mod.rs +├── client.rs # Docker client wrapper +├── containers.rs # Container operations +└── types.rs # Type conversions +``` + +--- + +## Afternoon: Container Management + +### Tasks +- [ ] Implement container listing +- [ ] Implement quarantine (disconnect network) +- [ ] Implement release (reconnect network) +- [ ] Cache container data in DB + +--- + +## Success Criteria + +- [ ] Can list real Docker containers +- [ ] Can get container details +- [ ] Quarantine actually disconnects network +- [ ] Release reconnects network +- [ ] All tests passing + +--- + +*Plan created: 2026-03-16* diff --git a/docs/DAY2_PROGRESS.md b/docs/DAY2_PROGRESS.md new file mode 100644 index 0000000..002faef --- /dev/null +++ b/docs/DAY2_PROGRESS.md @@ -0,0 +1,126 @@ +# Day 2 Progress Report - Docker Integration + +**Date:** 2026-03-16 +**Status:** ⚠️ Partial Progress + +--- + +## What Was Accomplished + +### ✅ Docker Module Structure Created +- `src/docker/client.rs` - Docker client wrapper +- `src/docker/containers.rs` - Container management +- `src/docker/mod.rs` - Module exports + +### ✅ Docker Client Implementation +- Connection to Docker daemon +- List containers +- Get container info +- Quarantine (disconnect networks) +- Release (reconnect) + +### ✅ Container Manager +- High-level container operations +- Alert generation on quarantine +- Security status calculation + +### ✅ Containers API +- `GET /api/containers` - List containers +- `POST /api/containers/:id/quarantine` - Quarantine container +- `POST /api/containers/:id/release` - Release container +- Fallback to mock data if Docker unavailable + +--- + +## Current Blockers + +### Bollard Crate Linking +The bollard crate isn't linking properly in the binary. + +**Errors:** +- `can't find crate for bollard` +- Type annotation issues in API handlers + +**Possible Causes:** +1. Bollard needs to be in lib.rs extern crate +2. Version incompatibility +3. Feature flags needed + +--- + +## Files Created (4 files) + +### Docker Module +- `src/docker/client.rs` (176 lines) +- `src/docker/containers.rs` (144 lines) +- `src/docker/mod.rs` (8 lines) + +### API +- `src/api/containers.rs` (updated, 168 lines) + +### Documentation +- `docs/DAY2_PLAN.md` +- `docs/DAY2_PROGRESS.md` + +--- + +## Time Spent + +| Task | Time | +|------|------| +| Docker client implementation | 1.5 hours | +| Container manager | 1 hour | +| Containers API | 1 hour | +| Debugging bollard linking | 1.5 hours | +| **Total** | **5 hours** | + +--- + +## Remaining Work + +### To Complete Docker Integration +1. Fix bollard crate linking (30 min) +2. Test with real Docker daemon (30 min) +3. Add container security scanning (1 hour) +4. Add threat detection rules (1 hour) + +**Estimated time:** 3 hours + +--- + +## Recommended Next Steps + +### Option A: Fix Bollard Linking (Recommended) +Add bollard to lib.rs: +```rust +#[cfg(target_os = "linux")] +extern crate bollard; +``` + +Then fix type annotations in API handlers. + +### Option B: Use Docker CLI Instead +Use `std::process::Command` to run docker commands: +```rust +Command::new("docker").arg("ps").output() +``` + +Simpler but less elegant. + +### Option C: Mock for Now +Keep mock data, implement real Docker later. + +--- + +## Decision Point + +**Choose one:** +1. **Fix bollard** - Continue with current approach (30 min) +2. **Use docker CLI** - Switch to command-line approach +3. **Mock for now** - Focus on other features + +**Recommendation:** Option 1 - Fix bollard linking, it's almost working. + +--- + +*Report generated: 2026-03-16* diff --git a/docs/REAL_FUNCTIONALITY_PLAN.md b/docs/REAL_FUNCTIONALITY_PLAN.md new file mode 100644 index 0000000..a5ebdbd --- /dev/null +++ b/docs/REAL_FUNCTIONALITY_PLAN.md @@ -0,0 +1,410 @@ +# Real Functionality Implementation Plan + +**Goal:** Add real Docker integration and database persistence +**Timeline:** 3-5 days +**Target Release:** v0.3.0 "Alpha" + +--- + +## Day 1: Database Integration + +### Morning: SQLite Schema & Migrations + +**Tasks:** +1. Create database schema +2. Write SQL migrations +3. Test migration execution + +**Files:** +``` +migrations/ +├── 00000000000000_create_alerts/ +│ ├── up.sql +│ └── down.sql +├── 00000000000001_create_threats/ +│ ├── up.sql +│ └── down.sql +└── 00000000000002_create_containers_cache/ + ├── up.sql + └── down.sql +``` + +**Schema:** +```sql +-- Alerts table +CREATE TABLE alerts ( + id TEXT PRIMARY KEY, + alert_type TEXT NOT NULL, + severity TEXT NOT NULL, + message TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'New', + timestamp DATETIME NOT NULL, + metadata TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Threats table +CREATE TABLE threats ( + id TEXT PRIMARY KEY, + threat_type TEXT NOT NULL, + severity TEXT NOT NULL, + score INTEGER NOT NULL, + source TEXT NOT NULL, + timestamp DATETIME NOT NULL, + status TEXT NOT NULL DEFAULT 'New', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Containers cache table +CREATE TABLE containers_cache ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + image TEXT NOT NULL, + status TEXT NOT NULL, + risk_score INTEGER DEFAULT 0, + last_updated DATETIME DEFAULT CURRENT_TIMESTAMP +); +``` + +**Tests:** +- Migration runs successfully +- Tables created correctly +- Can insert/query data + +--- + +### Afternoon: Database Repository Layer + +**Tasks:** +1. Create repository traits +2. Implement AlertRepository +3. Implement ThreatRepository +4. Implement ContainerRepository + +**Files:** +``` +src/database/ +├── mod.rs +├── connection.rs # DB connection pool +├── repositories/ +│ ├── mod.rs +│ ├── alerts.rs +│ ├── threats.rs +│ └── containers.rs +└── models/ + ├── mod.rs + ├── alert.rs + ├── threat.rs + └── container.rs +``` + +**Implementation:** +```rust +// src/database/repositories/alerts.rs +pub trait AlertRepository: Send + Sync { + async fn list(&self, filter: AlertFilter) -> Result>; + async fn get(&self, id: &str) -> Result>; + async fn create(&self, alert: Alert) -> Result; + async fn update_status(&self, id: &str, status: AlertStatus) -> Result<()>; + async fn get_stats(&self) -> Result; +} +``` + +**Tests:** +- Can create alert +- Can list alerts with filter +- Can update status +- Stats calculation correct + +--- + +## Day 2: Docker Integration + +### Morning: Docker Client Setup + +**Tasks:** +1. Add bollard dependency +2. Create Docker client wrapper +3. Test Docker connection +4. List containers + +**Files:** +``` +src/docker/ +├── mod.rs +├── client.rs # Docker client wrapper +├── containers.rs # Container operations +└── types.rs # Docker type conversions +``` + +**Implementation:** +```rust +// src/docker/client.rs +pub struct DockerClient { + client: bollard::Docker, +} + +impl DockerClient { + pub fn new() -> Result; + pub async fn list_containers(&self) -> Result>; + pub async fn get_container(&self, id: &str) -> Result; + pub async fn quarantine_container(&self, id: &str) -> Result<()>; + pub async fn release_container(&self, id: &str) -> Result<()>; +} +``` + +**Tests:** +- Docker client connects +- Can list containers +- Can get container details + +--- + +### Afternoon: Container Management + +**Tasks:** +1. Implement container listing +2. Implement quarantine (disconnect network) +3. Implement release (reconnect network) +4. Cache container data in DB + +**Implementation:** +```rust +// Quarantine implementation +pub async fn quarantine_container(&self, id: &str) -> Result<()> { + // Disconnect from all networks + let networks = self.client.list_networks().await?; + for network in networks { + self.client.disconnect_network( + &network.name, + NetworkDisconnectOptions { + container_id: Some(id.to_string()), + ..Default::default() + } + ).await?; + } + Ok(()) +} +``` + +**Tests:** +- List real containers from Docker +- Quarantine actually disconnects network +- Release reconnects network + +--- + +## Day 3: Connect API to Real Data + +### Morning: Update API Endpoints + +**Tasks:** +1. Inject repositories into API handlers +2. Replace mock data with DB queries +3. Test all endpoints + +**Changes:** +```rust +// Before (mock) +pub async fn get_alerts() -> impl Responder { + let alerts = vec![/* mock data */]; + HttpResponse::Ok().json(alerts) +} + +// After (real) +pub async fn get_alerts( + repo: web::Data, + query: web::Query +) -> impl Responder { + let filter = AlertFilter::from(query); + let alerts = repo.list(filter).await?; + HttpResponse::Ok().json(alerts) +} +``` + +**Endpoints to Update:** +- [ ] `GET /api/alerts` - Query database +- [ ] `GET /api/alerts/stats` - Calculate from DB +- [ ] `POST /api/alerts/:id/acknowledge` - Update DB +- [ ] `POST /api/alerts/:id/resolve` - Update DB +- [ ] `GET /api/containers` - Query Docker + cache +- [ ] `POST /api/containers/:id/quarantine` - Call Docker API +- [ ] `POST /api/containers/:id/release` - Call Docker API +- [ ] `GET /api/threats` - Query database +- [ ] `GET /api/threats/statistics` - Calculate from DB + +--- + +### Afternoon: Testing & Bug Fixes + +**Tasks:** +1. Test each endpoint with real data +2. Fix any bugs +3. Add error handling +4. Performance testing + +**Test Script:** +```bash +# Test alerts endpoint +curl http://localhost:5000/api/alerts + +# Test containers endpoint +curl http://localhost:5000/api/containers + +# Test quarantine +curl -X POST http://localhost:5000/api/containers/test123/quarantine +``` + +--- + +## Day 4: Real-Time Events + +### Morning: Event Generation + +**Tasks:** +1. Create event generator service +2. Generate alerts from Docker events +3. Store events in database + +**Implementation:** +```rust +// Listen to Docker events +pub async fn listen_docker_events( + client: DockerClient, + alert_repo: Arc +) { + let mut events = client.events().await; + while let Some(event) = events.next().await { + match event { + DockerEvent::ContainerStart { id, name } => { + alert_repo.create(Alert::new( + AlertType::SystemEvent, + AlertSeverity::Info, + format!("Container {} started", name) + )).await?; + } + DockerEvent::ContainerDie { id, name } => { + // Check if container was quarantined + } + _ => {} + } + } +} +``` + +--- + +### Afternoon: WebSocket Real-Time Updates + +**Tasks:** +1. Implement proper WebSocket with actix-web-actors +2. Broadcast events to connected clients +3. Test real-time updates + +--- + +## Day 5: Polish & Release Prep + +### Morning: Security Features + +**Tasks:** +1. Add basic threat detection rules +2. Generate alerts from suspicious activity +3. Test detection accuracy + +**Example Rules:** +```rust +// Rule: Container running as root +if container.user == "root" { + generate_alert(AlertSeverity::Medium, "Container running as root"); +} + +// Rule: Container with privileged mode +if container.privileged { + generate_alert(AlertSeverity::High, "Container in privileged mode"); +} +``` + +--- + +### Afternoon: Release Preparation + +**Tasks:** +1. Update CHANGELOG.md +2. Update README.md with real features +3. Write release notes +4. Create git tag v0.3.0-alpha +5. Test release build + +--- + +## Success Criteria + +### Must Have (for v0.3.0-alpha) + +- [ ] Alerts stored in SQLite +- [ ] Can list real Docker containers +- [ ] Can actually quarantine container +- [ ] Can actually release container +- [ ] Alert acknowledge/resolve persists +- [ ] All API endpoints use real data + +### Nice to Have + +- [ ] Real-time WebSocket updates +- [ ] Docker event listening +- [ ] Basic threat detection rules +- [ ] Container risk scoring + +### Future (v0.4.0+) + +- [ ] eBPF syscall monitoring +- [ ] ML anomaly detection +- [ ] Advanced threat detection +- [ ] Network traffic analysis + +--- + +## Risks & Mitigations + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Docker API changes | Medium | Use stable bollard version | +| SQLite concurrency | Low | Use connection pool | +| WebSocket complexity | Medium | Use polling as fallback | +| Performance issues | Medium | Add caching layer | + +--- + +## Testing Checklist + +### Database +- [ ] Migrations run successfully +- [ ] Can insert alerts +- [ ] Can query alerts with filters +- [ ] Can update alert status +- [ ] Stats calculation correct + +### Docker +- [ ] Can list containers +- [ ] Can get container details +- [ ] Quarantine disconnects network +- [ ] Release reconnects network +- [ ] Works with running containers + +### API +- [ ] All endpoints return real data +- [ ] Error handling works +- [ ] CORS works +- [ ] Performance acceptable + +### Frontend +- [ ] Dashboard shows real containers +- [ ] Can acknowledge alerts +- [ ] Can resolve alerts +- [ ] Quarantine button works +- [ ] Release button works + +--- + +*Plan created: 2026-03-15* diff --git a/docs/tasks/TASK-001-SUMMARY.md b/docs/tasks/TASK-001-SUMMARY.md deleted file mode 100644 index 83a291e..0000000 --- a/docs/tasks/TASK-001-SUMMARY.md +++ /dev/null @@ -1,225 +0,0 @@ -# TASK-001 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Project Structure Created - -All security-focused module directories and files have been created: - -``` -stackdog/ -├── src/ -│ ├── collectors/ ✅ Complete -│ │ ├── ebpf/ -│ │ │ ├── mod.rs -│ │ │ ├── loader.rs -│ │ │ └── programs/ -│ │ ├── docker_events.rs -│ │ └── network.rs -│ ├── events/ ✅ Complete -│ │ ├── syscall.rs -│ │ └── security.rs -│ ├── rules/ ✅ Complete -│ │ ├── engine.rs -│ │ ├── rule.rs -│ │ └── signatures.rs -│ ├── ml/ ✅ Stub created -│ ├── firewall/ ✅ Stub created -│ ├── response/ ✅ Stub created -│ ├── correlator/ ✅ Stub created -│ ├── alerting/ ✅ Stub created -│ ├── baselines/ ✅ Stub created -│ ├── database/ ✅ Stub created -│ └── main.rs ✅ Updated -├── ebpf/ ✅ Crate created -│ ├── Cargo.toml -│ └── src/ -├── tests/ ✅ Test structure created -│ ├── integration.rs -│ ├── events/ -│ ├── collectors/ -│ └── structure/ -└── benches/ ✅ Benchmark stubs created -``` - -### 2. ✅ Dependencies Updated (Cargo.toml) - -New dependencies added: -- **eBPF:** `aya = "0.12"`, `aya-obj = "0.1"` -- **ML:** `candle-core = "0.3"`, `candle-nn = "0.3"` -- **Firewall:** `netlink-packet-route = "0.17"`, `netlink-sys = "0.8"` -- **Testing:** `mockall = "0.11"`, `criterion = "0.5"` -- **Utilities:** `anyhow = "1"`, `thiserror = "1"` - -### 3. ✅ TDD Tests Created - -#### Module Structure Tests -- `tests/structure/mod_test.rs` - Verifies all modules can be imported - -#### Event Tests -- `tests/events/syscall_event_test.rs` - 12 tests for SyscallEvent -- `tests/events/security_event_test.rs` - 10 tests for SecurityEvent enum - -#### Collector Tests -- `tests/collectors/ebpf_loader_test.rs` - 5 tests for EbpfLoader - -### 4. ✅ Implementations with Tests - -#### SyscallEvent (`src/events/syscall.rs`) -- ✅ `SyscallType` enum with all syscall variants -- ✅ `SyscallEvent` struct with builder pattern -- ✅ Full test coverage (10 tests in module) -- ✅ Serialize/Deserialize support -- ✅ Debug, Clone, PartialEq derives - -#### Rule Engine (`src/rules/`) -- ✅ `Rule` trait with `evaluate()` method -- ✅ `RuleEngine` with priority-based ordering -- ✅ `Signature` and `SignatureDatabase` for threat detection -- ✅ Built-in signatures for crypto miners, container escape, network scanners - -#### eBPF Loader (`src/collectors/ebpf/loader.rs`) -- ✅ `EbpfLoader` struct -- ✅ Stub methods for TASK-003 implementation -- ✅ Unit tests included - -### 5. ✅ Documentation Created/Updated - -- ✅ **DEVELOPMENT.md** - Comprehensive 18-week development plan -- ✅ **CHANGELOG.md** - Updated with security focus -- ✅ **TODO.md** - Detailed task breakdown for all phases -- ✅ **BUGS.md** - Bug tracking template -- ✅ **QWEN.md** - Updated project context -- ✅ **.qwen/PROJECT_MEMORY.md** - Project memory and decisions -- ✅ **docs/tasks/TASK-001.md** - Detailed task specification - -### 6. ✅ eBPF Crate Created - -- ✅ `ebpf/Cargo.toml` with aya-ebpf dependency -- ✅ `.cargo/config` for BPF target -- ✅ Source structure for eBPF programs - ---- - -## Test Results - -### Tests Created - -| Test File | Tests Count | Status | -|-----------|-------------|--------| -| `tests/structure/mod_test.rs` | 10 | ✅ Compiles | -| `tests/events/syscall_event_test.rs` | 12 | ✅ Compiles | -| `tests/events/security_event_test.rs` | 11 | ✅ Compiles | -| `tests/collectors/ebpf_loader_test.rs` | 5 | ✅ Compiles | -| **Total** | **38** | | - -### Running Tests - -```bash -# Run all tests -cargo test --all - -# Run specific test modules -cargo test --test events::syscall_event_test -cargo test --test events::security_event_test -cargo test --test collectors::ebpf_loader_test - -# Run with coverage -cargo tarpaulin --all -``` - ---- - -## Code Quality - -### Clean Code Principles Applied - -1. **DRY** - Common patterns extracted (builder pattern, Default traits) -2. **Single Responsibility** - Each module has one purpose -3. **Open/Closed** - Traits for extensibility (Rule trait) -4. **Functional First** - Immutable data, From/Into ready -5. **Builder Pattern** - For complex object construction - -### Code Organization - -- Modules are flat (minimal nesting) -- Public APIs documented with `///` comments -- Test modules included in each source file -- Error handling with `anyhow::Result` - ---- - -## Next Steps (TASK-002) - -**TASK-002: Define Security Event Types** will: - -1. Expand event types with more fields -2. Add conversion traits (From/Into) -3. Implement event serialization -4. Add event validation -5. Create event stream types - ---- - -## Known Issues - -None. All code compiles successfully. - ---- - -## How to Continue - -### Option 1: Run Tests -```bash -cd /Users/vasilipascal/work/stackdog -cargo test --all -``` - -### Option 2: Start TASK-002 -See `TODO.md` for TASK-002 details. - -### Option 3: Build Project -```bash -cargo build -``` - ---- - -## Files Modified/Created - -### Created (40+ files) -- All module files in `src/collectors/`, `src/events/`, `src/rules/`, etc. -- All test files in `tests/` -- All documentation files -- eBPF crate files -- Benchmark files - -### Modified -- `Cargo.toml` - Updated dependencies -- `src/main.rs` - Added new module declarations -- `CHANGELOG.md` - Updated with security focus -- `QWEN.md` - Updated project context - ---- - -## Compliance Checklist - -- [x] All directories created -- [x] All module files compile -- [x] TDD tests created -- [x] `cargo fmt --all` ready -- [x] `cargo clippy --all` ready (pending full build) -- [x] Module structure tests verify imports -- [x] Event types have unit tests -- [x] Documentation comments for public APIs -- [x] Changelog updated - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-001.md b/docs/tasks/TASK-001.md deleted file mode 100644 index b323d79..0000000 --- a/docs/tasks/TASK-001.md +++ /dev/null @@ -1,609 +0,0 @@ -# Task Specification: TASK-001 - -## Create Project Structure for Security Modules - -**Phase:** 1 - Foundation & eBPF Collectors -**Priority:** High -**Estimated Effort:** 2-3 days -**Status:** 🟢 Ready for Development - ---- - -## Objective - -Create the new project directory structure for security-focused modules, update dependencies, and establish the eBPF build pipeline. This is the foundational task that enables all subsequent security feature development. - ---- - -## Requirements - -### 1. Directory Structure - -Create the following directory structure under `src/`: - -``` -src/ -├── collectors/ -│ ├── ebpf/ -│ │ ├── mod.rs -│ │ ├── loader.rs # eBPF program loader -│ │ └── programs/ # eBPF program definitions -│ │ └── mod.rs -│ ├── docker_events.rs -│ ├── network.rs -│ └── mod.rs -├── events/ -│ ├── mod.rs -│ ├── syscall.rs # SyscallEvent types -│ └── security.rs # SecurityEvent enum -├── rules/ -│ ├── mod.rs -│ ├── engine.rs # Rule evaluation engine -│ ├── rule.rs # Rule trait -│ └── signatures.rs # Known threat signatures -├── ml/ -│ ├── mod.rs -│ ├── candle_backend.rs -│ ├── features.rs -│ ├── anomaly.rs -│ ├── scorer.rs -│ └── models/ -│ ├── mod.rs -│ └── isolation_forest.rs -├── firewall/ -│ ├── mod.rs -│ ├── nftables.rs -│ ├── iptables.rs -│ └── quarantine.rs -├── response/ -│ ├── mod.rs -│ ├── actions.rs -│ └── pipeline.rs -├── correlator/ -│ ├── mod.rs -│ └── engine.rs -├── alerting/ -│ ├── mod.rs -│ ├── rules.rs -│ ├── notifications.rs -│ └── dedup.rs -├── baselines/ -│ ├── mod.rs -│ └── learning.rs -├── database/ -│ ├── mod.rs -│ ├── events.rs -│ └── baselines.rs -├── api/ # Existing - keep and update -├── config/ # Existing - keep -├── middleware/ # Existing - keep -├── models/ # Existing - keep -├── services/ # Existing - keep -├── utils/ # Existing - keep -├── constants.rs # Existing - keep -├── error.rs # Existing - update -├── main.rs # Existing - update -└── schema.rs # Existing - keep -``` - -### 2. Create `ebpf/` Crate - -Create a separate Cargo workspace member for eBPF programs: - -``` -ebpf/ -├── Cargo.toml -├── .cargo/ -│ └── config -└── src/ - ├── lib.rs - ├── syscalls.rs - └── maps.rs -``` - -### 3. Update `Cargo.toml` - -Add new dependencies for security features: - -```toml -[dependencies] -# eBPF -aya = "0.12" -aya-obj = "0.1" - -# ML -candle-core = "0.3" -candle-nn = "0.3" - -# Firewall -netlink-packet-route = "0.17" -netlink-sys = "0.8" - -# Existing dependencies (keep) -actix-web = "4" -# ... rest of existing deps -``` - -### 4. Create Module Files - -Each new module should have: -- `mod.rs` with module declaration -- Basic struct/enum definitions -- `#[cfg(test)]` test module stub - ---- - -## TDD Approach - -### Step 1: Write Tests First - -Create test files before implementation: - -#### Test 1: Module Structure Tests - -**File:** `tests/structure/mod_test.rs` - -```rust -/// Test that all security modules can be imported -#[test] -fn test_collectors_module_imports() { - // Verify collectors module exists and can be imported - use stackdog::collectors; - // Test passes if module compiles -} - -#[test] -fn test_events_module_imports() { - use stackdog::events; -} - -#[test] -fn test_rules_module_imports() { - use stackdog::rules; -} - -#[test] -fn test_ml_module_imports() { - use stackdog::ml; -} - -#[test] -fn test_firewall_module_imports() { - use stackdog::firewall; -} -``` - -#### Test 2: Event Type Tests - -**File:** `tests/events/syscall_event_test.rs` - -```rust -use stackdog::events::syscall::{SyscallEvent, SyscallType}; -use chrono::Utc; - -#[test] -fn test_syscall_event_creation() { - let event = SyscallEvent::new( - 1234, // pid - 1000, // uid - SyscallType::Execve, - Utc::now(), - ); - - assert_eq!(event.pid, 1234); - assert_eq!(event.uid, 1000); - assert_eq!(event.syscall_type, SyscallType::Execve); -} - -#[test] -fn test_syscall_event_builder() { - let event = SyscallEvent::builder() - .pid(1234) - .uid(1000) - .syscall_type(SyscallType::Execve) - .container_id(Some("abc123".to_string())) - .build(); - - assert_eq!(event.pid, 1234); - assert_eq!(event.container_id, Some("abc123".to_string())); -} -``` - -#### Test 3: eBPF Loader Tests - -**File:** `tests/collectors/ebpf_loader_test.rs` - -```rust -use stackdog::collectors::ebpf::loader::EbpfLoader; - -#[test] -fn test_ebpf_loader_creation() { - let loader = EbpfLoader::new(); - assert!(loader.is_ok()); -} - -#[test] -#[ignore] // Requires root and eBPF support -fn test_ebpf_program_load() { - let mut loader = EbpfLoader::new().unwrap(); - let result = loader.load_program("syscall_monitor"); - assert!(result.is_ok()); -} -``` - -### Step 2: Run Tests (Verify Failure) - -```bash -# Run tests - they should fail initially -cargo test --test structure::mod_test -cargo test --test events::syscall_event_test -cargo test --test collectors::ebpf_loader_test -``` - -### Step 3: Implement Minimal Code - -Implement just enough code to make tests pass: - -1. Create module files with basic structs -2. Implement `new()` and builder methods -3. Add `#[derive(Debug, Clone, PartialEq)]` where appropriate - -### Step 4: Verify Tests Pass - -```bash -# All tests should pass now -cargo test --test structure::mod_test -cargo test --test events::syscall_event_test -``` - -### Step 5: Refactor - -- Extract common code -- Apply DRY principle -- Add documentation comments -- Run `cargo fmt` and `cargo clippy` - ---- - -## Implementation Details - -### 1. Event Types (`src/events/syscall.rs`) - -```rust -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum SyscallType { - Execve, - Execveat, - Connect, - Accept, - Bind, - Open, - Openat, - Ptrace, - Setuid, - Setgid, - Mount, - Umount, - Unknown, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SyscallEvent { - pub pid: u32, - pub uid: u32, - pub syscall_type: SyscallType, - pub timestamp: DateTime, - pub container_id: Option, - pub comm: Option, -} - -impl SyscallEvent { - pub fn new( - pid: u32, - uid: u32, - syscall_type: SyscallType, - timestamp: DateTime, - ) -> Self { - Self { - pid, - uid, - syscall_type, - timestamp, - container_id: None, - comm: None, - } - } - - pub fn builder() -> SyscallEventBuilder { - SyscallEventBuilder::new() - } -} - -// Builder pattern -pub struct SyscallEventBuilder { - pid: u32, - uid: u32, - syscall_type: SyscallType, - timestamp: Option>, - container_id: Option, - comm: Option, -} - -impl SyscallEventBuilder { - pub fn new() -> Self { - Self { - pid: 0, - uid: 0, - syscall_type: SyscallType::Unknown, - timestamp: None, - container_id: None, - comm: None, - } - } - - pub fn pid(mut self, pid: u32) -> Self { - self.pid = pid; - self - } - - pub fn uid(mut self, uid: u32) -> Self { - self.uid = uid; - self - } - - pub fn syscall_type(mut self, syscall_type: SyscallType) -> Self { - self.syscall_type = syscall_type; - self - } - - pub fn timestamp(mut self, timestamp: DateTime) -> Self { - self.timestamp = Some(timestamp); - self - } - - pub fn container_id(mut self, container_id: Option) -> Self { - self.container_id = container_id; - self - } - - pub fn comm(mut self, comm: Option) -> Self { - self.comm = comm; - self - } - - pub fn build(self) -> SyscallEvent { - SyscallEvent { - pid: self.pid, - uid: self.uid, - syscall_type: self.syscall_type, - timestamp: self.timestamp.unwrap_or_else(Utc::now), - container_id: self.container_id, - comm: self.comm, - } - } -} - -impl Default for SyscallEventBuilder { - fn default() -> Self { - Self::new() - } -} -``` - -### 2. Security Event Enum (`src/events/security.rs`) - -```rust -use crate::events::syscall::SyscallEvent; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum SecurityEvent { - Syscall(SyscallEvent), - Network(NetworkEvent), - Container(ContainerEvent), - Alert(AlertEvent), -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct NetworkEvent { - pub src_ip: String, - pub dst_ip: String, - pub src_port: u16, - pub dst_port: u16, - pub protocol: String, - pub timestamp: DateTime, - pub container_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ContainerEvent { - pub container_id: String, - pub event_type: ContainerEventType, - pub timestamp: DateTime, - pub details: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum ContainerEventType { - Start, - Stop, - Create, - Destroy, - Pause, - Unpause, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AlertEvent { - pub alert_type: AlertType, - pub severity: AlertSeverity, - pub message: String, - pub timestamp: DateTime, - pub source_event_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum AlertType { - ThreatDetected, - AnomalyDetected, - RuleViolation, - QuarantineApplied, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum AlertSeverity { - Info, - Low, - Medium, - High, - Critical, -} -``` - -### 3. eBPF Loader (`src/collectors/ebpf/loader.rs`) - -```rust -use anyhow::Result; -use aya::{Bpf, BpfLoader}; - -pub struct EbpfLoader { - bpf: Option, -} - -impl EbpfLoader { - pub fn new() -> Result { - Ok(Self { bpf: None }) - } - - pub fn load_program(&mut self, program_name: &str) -> Result<()> { - // Implementation will be added in TASK-003 - Ok(()) - } -} - -impl Default for EbpfLoader { - fn default() -> Self { - Self::new().unwrap() - } -} -``` - ---- - -## Acceptance Criteria - -- [ ] All new directories created -- [ ] All module files compile without errors -- [ ] All TDD tests pass -- [ ] `cargo fmt --all` produces no changes -- [ ] `cargo clippy --all` produces no warnings -- [ ] Module structure tests verify imports work -- [ ] Event types have unit tests with 100% coverage -- [ ] Documentation comments for public APIs -- [ ] Changelog updated - ---- - -## Test Commands - -```bash -# Run structure tests -cargo test --test structure::mod_test - -# Run event tests -cargo test --test events::syscall_event_test -cargo test --test events::security_event_test - -# Run eBPF loader tests -cargo test --test collectors::ebpf_loader_test - -# Run all tests -cargo test --all - -# Check formatting -cargo fmt --all -- --check - -# Check for clippy warnings -cargo clippy --all -``` - ---- - -## Dependencies - -### Required Crates - -Add to `Cargo.toml`: - -```toml -[dependencies] -# eBPF -aya = "0.12" -aya-obj = "0.1" - -# ML (prepare for future tasks) -candle-core = "0.3" -candle-nn = "0.3" - -# Firewall (prepare for future tasks) -netlink-packet-route = "0.17" -netlink-sys = "0.8" - -# Utilities -anyhow = "1" -thiserror = "1" -``` - -### Development Dependencies - -```toml -[dev-dependencies] -tokio-test = "0.4" -mockall = "0.11" -``` - ---- - -## Risks and Mitigations - -| Risk | Impact | Mitigation | -|------|--------|------------| -| eBPF kernel compatibility | Medium | Test on target kernel version, provide fallback | -| Directory structure complexity | Low | Keep structure flat, avoid over-nesting | -| Dependency conflicts | Low | Use compatible versions, test early | - ---- - -## Related Tasks - -- **TASK-002**: Define security event types (builds on this task) -- **TASK-003**: Setup aya-rs eBPF integration (builds on this task) -- **TASK-004**: Implement syscall event capture (builds on TASK-003) - ---- - -## Resources - -- [Rust Module System](https://doc.rust-lang.org/book/ch07-00-managing-growing-projects-with-packages-crates-and-modules.html) -- [Builder Pattern in Rust](https://rust-unofficial.github.io/patterns/patterns/creational/builder.html) -- [aya-rs Documentation](https://aya-rs.dev/) -- [Candle Documentation](https://docs.rs/candle-core) - ---- - -## Notes - -- Start with minimal implementation to pass tests -- Refactor after tests pass -- Keep functions small and focused -- Use `#[derive]` macros for common traits -- Document public APIs with `///` comments - ---- - -*Created: 2026-03-13* -*Last Updated: 2026-03-13* diff --git a/docs/tasks/TASK-002-SUMMARY.md b/docs/tasks/TASK-002-SUMMARY.md deleted file mode 100644 index ae573fa..0000000 --- a/docs/tasks/TASK-002-SUMMARY.md +++ /dev/null @@ -1,221 +0,0 @@ -# TASK-002 Implementation Summary - -**Status:** ✅ **COMPLETE** (Core Implementation) -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Event Types Fully Implemented - -#### SyscallEvent (`src/events/syscall.rs`) -- ✅ Complete `SyscallType` enum with all variants -- ✅ `SyscallEvent` struct with full builder pattern -- ✅ `From`/`Into` traits for `SecurityEvent` conversion -- ✅ `pid()` and `uid()` helper methods -- ✅ Serialize/Deserialize with serde -- ✅ Debug, Clone, PartialEq derives -- ✅ Built-in unit tests - -#### SecurityEvent (`src/events/security.rs`) -- ✅ Complete enum with Syscall, Network, Container, Alert variants -- ✅ `From` implementations for all event types -- ✅ `pid()`, `uid()`, `timestamp()` helper methods -- ✅ Full serialization support - -#### Event Validation (`src/events/validation.rs`) -- ✅ `ValidationResult` enum (Valid, Invalid, Error) -- ✅ `EventValidator` with methods: - - `validate_syscall()` - - `validate_network()` - IP address validation - - `validate_alert()` - message validation - - `validate_ip()` - standalone IP validation - - `validate_port()` - port validation -- ✅ Display trait implementation - -#### Event Stream Types (`src/events/stream.rs`) -- ✅ `EventBatch` - batch processing with add/clear/iter -- ✅ `EventFilter` - fluent filter builder with: - - `with_syscall_type()` - - `with_pid()` - - `with_uid()` - - `with_time_range()` - - `matches()` method -- ✅ `EventIterator` - streaming with filter support -- ✅ `FilteredEventIterator` - filtered iteration - -### 2. ✅ TDD Tests Created (50+ tests) - -| Test File | Tests | Status | -|-----------|-------|--------| -| `tests/events/event_conversion_test.rs` | 7 | ✅ Complete | -| `tests/events/event_serialization_test.rs` | 8 | ✅ Complete | -| `tests/events/event_validation_test.rs` | 12 | ✅ Complete | -| `tests/events/event_stream_test.rs` | 14 | ✅ Complete | -| `tests/events/syscall_event_test.rs` | 12 | ✅ Complete | -| `tests/events/security_event_test.rs` | 11 | ✅ Complete | -| **Total** | **64** | | - -### 3. ✅ Module Structure - -``` -src/events/ -├── mod.rs ✅ Updated with all submodules -├── syscall.rs ✅ Complete implementation -├── security.rs ✅ Complete implementation -├── validation.rs ✅ Complete implementation -└── stream.rs ✅ Complete implementation -``` - -### 4. ✅ Code Quality - -- **DRY Principle**: Common patterns extracted (builder pattern) -- **Functional Programming**: Immutable data, From/Into traits -- **Clean Code**: Functions < 50 lines, single responsibility -- **Documentation**: All public APIs documented with `///` - ---- - -## Test Results - -**Note:** Full compilation is blocked by dependency conflicts between: -- `actix-http` (requires older Rust const evaluation) -- `candle-core` (rand version conflicts) -- `aya` (Linux-only, macOS compatibility issues) - -### Workaround - -The events module code is complete and correct. Tests can be run in isolation: - -```bash -# When dependencies are resolved: -cargo test --test integration::events::event_conversion_test -cargo test --test integration::events::event_serialization_test -cargo test --test integration::events::event_validation_test -cargo test --test integration::events::event_stream_test -``` - ---- - -## Implementation Highlights - -### Event Conversion Example - -```rust -// Automatic conversion via From trait -let syscall_event = SyscallEvent::new(1234, 1000, SyscallType::Execve, Utc::now()); -let security_event: SecurityEvent = syscall_event.into(); - -// Pattern matching -match security_event { - SecurityEvent::Syscall(e) => println!("Syscall from PID {}", e.pid), - _ => {} -} -``` - -### Event Validation Example - -```rust -let event = NetworkEvent { /* ... */ }; -let result = EventValidator::validate_network(&event); - -if result.is_valid() { - println!("Event is valid"); -} else { - println!("Invalid: {}", result); -} -``` - -### Event Stream Example - -```rust -// Create batch -let mut batch = EventBatch::new(); -batch.add(event1); -batch.add(event2); - -// Filter events -let filter = EventFilter::new() - .with_syscall_type(SyscallType::Execve) - .with_pid(1234); - -let iterator = EventIterator::new(events); -let filtered: Vec<_> = iterator.filter(&filter).collect(); -``` - ---- - -## Known Issues - -### Dependency Conflicts (External) - -1. **actix-http** - Incompatible with newer Rust const evaluation -2. **candle-core** - rand crate version conflicts -3. **aya** - Linux-only, macOS compatibility issues - -### Resolution Path - -These are external dependency issues, not code issues. Resolution options: - -1. **Option A**: Use older Rust toolchain (1.70) -2. **Option B**: Wait for upstream fixes -3. **Option C**: Replace problematic dependencies - ---- - -## Next Steps - -### Immediate (TASK-003) - -Implement eBPF syscall monitoring: -1. Create eBPF programs in `ebpf/src/syscalls.rs` -2. Implement loader in `src/collectors/ebpf/loader.rs` -3. Add tracepoint attachments - -### Short Term - -1. Resolve dependency conflicts -2. Run full test suite -3. Add more integration tests - ---- - -## Files Modified/Created - -### Created (10 files) -- `src/events/mod.rs` - Module declaration -- `src/events/syscall.rs` - SyscallEvent implementation -- `src/events/security.rs` - SecurityEvent implementation -- `src/events/validation.rs` - Validation logic -- `src/events/stream.rs` - Stream types -- `tests/events/event_conversion_test.rs` - Conversion tests -- `tests/events/event_serialization_test.rs` - Serialization tests -- `tests/events/event_validation_test.rs` - Validation tests -- `tests/events/event_stream_test.rs` - Stream tests -- `docs/tasks/TASK-002.md` - Task specification - -### Modified -- `src/lib.rs` - Added library root -- `tests/integration.rs` - Updated test harness -- `tests/events/mod.rs` - Added new test modules -- `Cargo.toml` - Updated dependencies - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| All From/Into traits implemented | ✅ Complete | -| JSON serialization working | ✅ Complete (code ready) | -| Event validation implemented | ✅ Complete | -| Event stream types implemented | ✅ Complete | -| All tests passing | ⏳ Blocked by dependencies | -| 100% test coverage for event types | ✅ Code complete | -| Documentation complete | ✅ Complete | - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-002.md b/docs/tasks/TASK-002.md deleted file mode 100644 index 74b9d03..0000000 --- a/docs/tasks/TASK-002.md +++ /dev/null @@ -1,119 +0,0 @@ -# Task Specification: TASK-002 - -## Define Security Event Types - -**Phase:** 1 - Foundation & eBPF Collectors -**Priority:** High -**Estimated Effort:** 1-2 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Complete the security event types implementation with proper conversions, serialization, validation, and event stream support. This task builds on TASK-001's foundation. - ---- - -## Requirements - -### 1. Implement From/Into Traits - -Create conversions between: -- `SyscallEvent` ↔ `SecurityEvent` -- `NetworkEvent` ↔ `SecurityEvent` -- `ContainerEvent` ↔ `SecurityEvent` -- `AlertEvent` ↔ `SecurityEvent` -- Raw eBPF data → `SyscallEvent` - -### 2. Event Serialization - -- JSON serialization/deserialization -- Binary serialization for efficient storage -- Event ID generation (UUID) -- Timestamp handling - -### 3. Event Validation - -- Validate required fields -- Validate IP addresses -- Validate syscall types -- Validate severity levels - -### 4. Event Stream Types - -- Event batch for bulk operations -- Event filter for querying -- Event iterator for streaming - ---- - -## TDD Tests to Create - -### Test File: `tests/events/event_conversion_test.rs` - -```rust -#[test] -fn test_syscall_event_to_security_event() -#[test] -fn test_network_event_to_security_event() -#[test] -fn test_container_event_to_security_event() -#[test] -fn test_alert_event_to_security_event() -#[test] -fn test_security_event_into_syscall() -``` - -### Test File: `tests/events/event_serialization_test.rs` - -```rust -#[test] -fn test_syscall_event_json_serialize() -#[test] -fn test_syscall_event_json_deserialize() -#[test] -fn test_security_event_json_roundtrip() -#[test] -fn test_event_with_uuid() -``` - -### Test File: `tests/events/event_validation_test.rs` - -```rust -#[test] -fn test_valid_syscall_event() -#[test] -fn test_invalid_ip_address() -#[test] -fn test_invalid_severity() -#[test] -fn test_event_validation_result() -``` - -### Test File: `tests/events/event_stream_test.rs` - -```rust -#[test] -fn test_event_batch_creation() -#[test] -fn test_event_filter_matching() -#[test] -fn test_event_iterator() -``` - ---- - -## Acceptance Criteria - -- [ ] All From/Into traits implemented -- [ ] JSON serialization working -- [ ] Event validation implemented -- [ ] Event stream types implemented -- [ ] All tests passing (target: 25+ tests) -- [ ] 100% test coverage for event types -- [ ] Documentation complete - ---- - -*Created: 2026-03-13* diff --git a/docs/tasks/TASK-003-SUMMARY.md b/docs/tasks/TASK-003-SUMMARY.md deleted file mode 100644 index 8ba0aa1..0000000 --- a/docs/tasks/TASK-003-SUMMARY.md +++ /dev/null @@ -1,388 +0,0 @@ -# TASK-003 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ eBPF Loader Implementation - -**File:** `src/collectors/ebpf/loader.rs` - -#### Features Implemented -- `EbpfLoader` struct with full lifecycle management -- `load_program_from_bytes()` - Load from ELF bytes -- `load_program_from_file()` - Load from ELF file -- `attach_program()` - Attach to tracepoints -- `detach_program()` - Detach programs -- `unload_program()` - Unload programs -- `loaded_program_count()` - Program counting -- `is_program_loaded()` - Status checking -- `is_program_attached()` - Attachment status - -#### Error Handling -```rust -pub enum LoadError { - ProgramNotFound(String), - LoadFailed(String), - AttachFailed(String), - KernelVersionTooLow { required, current }, - NotLinux, - PermissionDenied, - Other(anyhow::Error), -} -``` - -#### Kernel Compatibility -- Automatic kernel version detection -- Checks for eBPF support (requires 4.19+) -- Graceful error on non-Linux platforms -- Feature-gated compilation - ---- - -### 2. ✅ Kernel Compatibility Module - -**File:** `src/collectors/ebpf/kernel.rs` - -#### KernelVersion Struct -```rust -pub struct KernelVersion { - pub major: u32, - pub minor: u32, - pub patch: u32, -} -``` - -**Methods:** -- `parse(&str) -> Result` - Parse version strings -- `meets_minimum(&KernelVersion) -> bool` - Version comparison -- `supports_ebpf() -> bool` - Check 4.19+ requirement -- `supports_btf() -> bool` - Check BTF support (5.4+) - -#### KernelInfo Struct -```rust -pub struct KernelInfo { - pub version: KernelVersion, - pub os: String, - pub arch: String, -} -``` - -**Methods:** -- `new() -> Result` - Get current kernel info -- `supports_ebpf() -> bool` - Check eBPF support -- `supports_btf() -> bool` - Check BTF support - -#### Utility Functions -- `check_kernel_version() -> Result` -- `get_kernel_version() -> Result` (Linux only) -- `is_linux() -> bool` - ---- - -### 3. ✅ Syscall Monitor - -**File:** `src/collectors/ebpf/syscall_monitor.rs` - -#### SyscallMonitor Struct -```rust -pub struct SyscallMonitor { - running: bool, - event_buffer: Vec, - // eBPF loader (Linux only) -} -``` - -**Methods:** -- `new() -> Result` - Create monitor -- `start() -> Result<()>` - Start monitoring -- `stop() -> Result<()>` - Stop monitoring -- `is_running() -> bool` - Check status -- `poll_events() -> Vec` - Poll for events -- `peek_events() -> &[SyscallEvent>` - Peek without consuming - ---- - -### 4. ✅ Event Ring Buffer - -**File:** `src/collectors/ebpf/ring_buffer.rs` - -#### EventRingBuffer Struct -```rust -pub struct EventRingBuffer { - buffer: Vec, - capacity: usize, -} -``` - -**Methods:** -- `new() -> Self` - Default capacity (4096) -- `with_capacity(usize) -> Self` - Custom capacity -- `push(SyscallEvent)` - Add event (FIFO overflow) -- `drain() -> Vec` - Get and clear -- `len() -> usize` - Event count -- `is_empty() -> bool` - Empty check -- `capacity() -> usize` - Get capacity -- `clear() -> Self` - Clear buffer - -**Features:** -- Automatic overflow handling (removes oldest) -- Efficient draining -- Configurable capacity - ---- - -### 5. ✅ eBPF Programs Module - -**File:** `src/collectors/ebpf/programs.rs` - -#### ProgramType Enum -```rust -pub enum ProgramType { - SyscallTracepoint, - NetworkMonitor, - ContainerMonitor, -} -``` - -#### ProgramMetadata Struct -```rust -pub struct ProgramMetadata { - pub name: &'static str, - pub program_type: ProgramType, - pub description: &'static str, - pub required_kernel: (u32, u32), -} -``` - -#### Built-in Programs -```rust -pub mod builtin { - pub const EXECVE_PROGRAM: ProgramMetadata; // execve monitoring - pub const CONNECT_PROGRAM: ProgramMetadata; // connect monitoring - pub const OPENAT_PROGRAM: ProgramMetadata; // openat monitoring - pub const PTRACE_PROGRAM: ProgramMetadata; // ptrace monitoring -} -``` - ---- - -## Tests Created - -### Test Files (3 files, 35+ tests) - -| Test File | Tests | Status | -|-----------|-------|--------| -| `tests/collectors/ebpf_loader_test.rs` | 8 | ✅ Complete | -| `tests/collectors/ebpf_syscall_test.rs` | 8 | ✅ Complete | -| `tests/collectors/ebpf_kernel_test.rs` | 10 | ✅ Complete | -| **Module Tests** | 9+ | ✅ Complete | -| **Total** | **35+** | | - -### Test Coverage - -#### Kernel Module Tests -```rust -test_kernel_version_parse() -test_kernel_version_parse_with_suffix() -test_kernel_version_parse_invalid() -test_kernel_version_comparison() -test_kernel_version_meets_minimum() -test_kernel_info_creation() -test_kernel_version_check_function() -test_kernel_version_display() -test_kernel_version_equality() -test_kernel_version_supports_ebpf() -test_kernel_version_supports_btf() -``` - -#### Loader Module Tests -```rust -test_ebpf_loader_creation() -test_ebpf_loader_default() -test_ebpf_loader_has_programs() -test_ebpf_program_load_success() (requires root) -test_ebpf_loader_error_display() -test_ebpf_loader_creation_cross_platform() -test_ebpf_is_linux_check() -``` - -#### Ring Buffer Tests -```rust -test_ring_buffer_creation() -test_ring_buffer_with_capacity() -test_ring_buffer_push() -test_ring_buffer_drain() -test_ring_buffer_overflow() -test_ring_buffer_clear() -``` - -#### Programs Module Tests -```rust -test_program_type_variants() -test_builtin_programs() -test_program_metadata() -``` - ---- - -## Module Structure - -``` -src/collectors/ebpf/ -├── mod.rs ✅ Module exports -├── loader.rs ✅ Program loader -├── kernel.rs ✅ Kernel compatibility -├── syscall_monitor.rs ✅ Syscall monitoring -├── ring_buffer.rs ✅ Event buffering -└── programs.rs ✅ Program definitions -``` - ---- - -## Code Quality - -### Cross-Platform Support -- ✅ Feature-gated compilation (`#[cfg(target_os = "linux")]`) -- ✅ Graceful degradation on non-Linux -- ✅ Clear error messages for unsupported platforms - -### Error Handling -- ✅ Custom error types with `thiserror` -- ✅ Contextual error messages -- ✅ Proper error propagation with `anyhow` - -### Documentation -- ✅ All public APIs documented with `///` -- ✅ Module-level documentation -- ✅ Example code in doc comments - ---- - -## Integration Points - -### With Event System -```rust -use crate::collectors::SyscallMonitor; -use crate::events::syscall::{SyscallEvent, SyscallType}; - -let mut monitor = SyscallMonitor::new()?; -monitor.start()?; - -let events = monitor.poll_events(); -for event in events { - // Process SyscallEvent -} -``` - -### With Rules Engine -```rust -let events = monitor.poll_events(); -for event in events { - let results = rule_engine.evaluate(&SecurityEvent::Syscall(event)); - // Handle rule matches -} -``` - ---- - -## Dependencies - -### Added -- `thiserror = "1"` - Error handling -- `log = "0.4"` - Logging - -### Existing (used) -- `anyhow = "1"` - Error context -- `chrono = "0.4"` - Timestamps - -### Required at Runtime (Linux only) -- `aya = "0.12"` - eBPF framework -- Kernel 4.19+ with eBPF support - ---- - -## Known Limitations - -### Current State -1. **Stub Implementation**: The loader and monitor are structurally complete but use stubs for actual eBPF operations -2. **No Real eBPF Programs**: Programs module defines metadata but actual eBPF code comes in TASK-004 -3. **Ring Buffer**: Uses Vec instead of actual eBPF ring buffer (will be replaced in TASK-004) - -### Next Steps (TASK-004) -1. Implement actual eBPF programs in `ebpf/src/syscalls.rs` -2. Connect ring buffer to eBPF perf buffer -3. Implement real syscall event capture -4. Add BTF support - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| eBPF loader compiles without errors | ✅ Complete | -| Programs load successfully on Linux 4.19+ | ✅ Structure ready | -| Syscall events captured and sent to userspace | ⏳ Stub ready | -| Ring buffer polling works correctly | ✅ Implemented | -| All tests passing (target: 15+ tests) | ✅ 35+ tests | -| Documentation complete | ✅ Complete | -| Error handling for non-Linux platforms | ✅ Complete | - ---- - -## Files Modified/Created - -### Created (8 files) -- `src/collectors/ebpf/loader.rs` - Program loader -- `src/collectors/ebpf/kernel.rs` - Kernel compatibility -- `src/collectors/ebpf/syscall_monitor.rs` - Syscall monitor -- `src/collectors/ebpf/ring_buffer.rs` - Event ring buffer -- `src/collectors/ebpf/programs.rs` - Program definitions -- `tests/collectors/ebpf_loader_test.rs` - Loader tests -- `tests/collectors/ebpf_syscall_test.rs` - Syscall tests -- `tests/collectors/ebpf_kernel_test.rs` - Kernel tests - -### Modified -- `src/collectors/ebpf/mod.rs` - Updated exports -- `src/collectors/mod.rs` - Added re-exports -- `src/lib.rs` - Added re-exports -- `tests/collectors/mod.rs` - Added test modules -- `Cargo.toml` - Already has dependencies - ---- - -## Usage Example - -```rust -use stackdog::collectors::{EbpfLoader, SyscallMonitor}; - -// Check kernel support -let loader = EbpfLoader::new()?; -if !loader.is_ebpf_supported() { - println!("eBPF not supported on this system"); - return; -} - -// Create and start monitor -let mut monitor = SyscallMonitor::new()?; -monitor.start()?; - -// Poll for events -loop { - let events = monitor.poll_events(); - for event in events { - println!("Syscall: {:?} from PID {}", - event.syscall_type, event.pid); - } - std::thread::sleep(std::time::Duration::from_millis(100)); -} -``` - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-003.md b/docs/tasks/TASK-003.md deleted file mode 100644 index 120741e..0000000 --- a/docs/tasks/TASK-003.md +++ /dev/null @@ -1,154 +0,0 @@ -# Task Specification: TASK-003 - -## Setup aya-rs eBPF Integration - -**Phase:** 1 - Foundation & eBPF Collectors -**Priority:** High -**Estimated Effort:** 2-3 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Implement the eBPF infrastructure using aya-rs framework. This includes the eBPF program loader, syscall tracepoint programs, and event ring buffer for sending events to userspace. - ---- - -## Requirements - -### 1. eBPF Program Loader - -- Load eBPF programs from ELF files -- Attach programs to kernel tracepoints -- Manage program lifecycle (load/unload) -- Error handling for unsupported kernels - -### 2. Syscall Tracepoint Programs - -Implement eBPF programs for: -- `sys_enter_execve` - Process execution -- `sys_enter_connect` - Network connections -- `sys_enter_openat` - File access -- `sys_enter_ptrace` - Debugging attempts - -### 3. Event Ring Buffer - -- Send events from eBPF to userspace -- Efficient event buffering -- Handle event loss gracefully - -### 4. Kernel Compatibility - -- Check kernel version (4.19+ required) -- Check BTF support -- Fallback mechanisms for older kernels - ---- - -## TDD Tests to Create - -### Test File: `tests/collectors/ebpf_loader_test.rs` - -```rust -#[test] -fn test_ebpf_loader_creation() -#[test] -fn test_ebpf_program_load_success() -#[test] -fn test_ebpf_program_load_not_found() -#[test] -fn test_ebpf_program_attach() -#[test] -fn test_ebpf_program_detach() -#[test] -fn test_ebpf_kernel_version_check() -``` - -### Test File: `tests/collectors/ebpf_syscall_test.rs` - -```rust -#[test] -fn test_execve_event_capture() -#[test] -fn test_connect_event_capture() -#[test] -fn test_openat_event_capture() -#[test] -fn test_ptrace_event_capture() -#[test] -fn test_event_ring_buffer_poll() -``` - -### Test File: `tests/collectors/ebpf_integration_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_full_ebpf_pipeline() -#[test] -#[ignore = "requires root"] -fn test_ebpf_event_to_userspace() -``` - ---- - -## Implementation Files - -### eBPF Programs (`ebpf/src/`) - -``` -ebpf/ -├── src/ -│ ├── lib.rs -│ ├── syscalls.rs # Syscall tracepoint programs -│ ├── maps.rs # eBPF maps (ring buffer, hash maps) -│ └── types.h # Shared C types for events -``` - -### Userspace Loader (`src/collectors/ebpf/`) - -``` -src/collectors/ebpf/ -├── mod.rs -├── loader.rs # Program loader -├── programs.rs # Program definitions -├── ring_buffer.rs # Event ring buffer -└── kernel.rs # Kernel compatibility -``` - ---- - -## Acceptance Criteria - -- [ ] eBPF loader compiles without errors -- [ ] Programs load successfully on Linux 4.19+ -- [ ] Syscall events captured and sent to userspace -- [ ] Ring buffer polling works correctly -- [ ] All tests passing (target: 15+ tests) -- [ ] Documentation complete -- [ ] Error handling for non-Linux platforms - ---- - -## Dependencies - -- `aya = "0.12"` - eBPF framework -- `aya-obj = "0.1"` - eBPF object loading -- `libc` - System calls -- `thiserror` - Error handling - ---- - -## Risks - -| Risk | Impact | Mitigation | -|------|--------|------------| -| Kernel < 4.19 | High | Version check, graceful fallback | -| No BTF support | Medium | Use non-BTF mode | -| Permission denied | High | Document root requirement | -| macOS development | High | Linux VM for testing | - ---- - -*Created: 2026-03-13* diff --git a/docs/tasks/TASK-004-SUMMARY.md b/docs/tasks/TASK-004-SUMMARY.md deleted file mode 100644 index d29993d..0000000 --- a/docs/tasks/TASK-004-SUMMARY.md +++ /dev/null @@ -1,414 +0,0 @@ -# TASK-004 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Test Suite Created (5 test files, 25+ tests) - -#### execve_capture_test.rs (5 tests) -- `test_execve_event_captured_on_process_spawn` -- `test_execve_event_contains_filename` -- `test_execve_event_contains_pid` -- `test_execve_event_contains_uid` -- `test_execve_event_timestamp` - -#### connect_capture_test.rs (4 tests) -- `test_connect_event_captured_on_tcp_connection` -- `test_connect_event_contains_destination_ip` -- `test_connect_event_contains_destination_port` -- `test_connect_event_multiple_connections` - -#### openat_capture_test.rs (4 tests) -- `test_openat_event_captured_on_file_open` -- `test_openat_event_contains_file_path` -- `test_openat_event_multiple_files` -- `test_openat_event_read_and_write` - -#### ptrace_capture_test.rs (3 tests) -- `test_ptrace_event_captured_on_trace_attempt` -- `test_ptrace_event_contains_target_pid` -- `test_ptrace_event_security_alert` - -#### event_enrichment_test.rs (13 tests) -- `test_event_enricher_creation` -- `test_enrich_adds_timestamp` -- `test_enrich_preserves_existing_timestamp` -- `test_container_detector_creation` -- `test_container_id_detection_format` -- `test_container_id_invalid_formats` -- `test_cgroup_parsing` -- `test_process_tree_enrichment` -- `test_process_comm_enrichment` -- `test_timestamp_normalization` -- `test_enrichment_pipeline` - ---- - -### 2. ✅ Event Enrichment Module - -**File:** `src/collectors/ebpf/enrichment.rs` - -#### EventEnricher Struct -```rust -pub struct EventEnricher { - process_cache: HashMap, -} -``` - -**Methods:** -- `new() -> Result` - Create enricher -- `enrich(&mut self, event: &mut SyscallEvent) -> Result<()>` - Full enrichment -- `get_parent_pid(pid: u32) -> Option` - Get parent PID -- `get_process_comm(pid: u32) -> Option` - Get process name -- `get_process_exe(pid: u32) -> Option` - Get executable path -- `get_process_cwd(pid: u32) -> Option` - Get working directory - -**Implementation Details:** -- Reads from `/proc/[pid]/stat` for parent PID -- Reads from `/proc/[pid]/comm` for command name -- Reads from `/proc/[pid]/cmdline` for full command -- Reads from `/proc/[pid]/exe` symlink for executable path -- Reads from `/proc/[pid]/cwd` symlink for working directory - ---- - -### 3. ✅ Container Detection Module - -**File:** `src/collectors/ebpf/container.rs` - -#### ContainerDetector Struct -```rust -pub struct ContainerDetector { - cache: HashMap, -} -``` - -**Methods:** -- `new() -> Result` - Create detector -- `detect_container(pid: u32) -> Option` - Detect for PID -- `current_container() -> Option` - Detect current process -- `validate_container_id(id: &str) -> bool` - Validate ID format -- `parse_container_from_cgroup(cgroup_line: &str) -> Option` - Parse cgroup - -**Container Detection Strategies:** - -1. **Docker Format** - ``` - 12:memory:/docker/abc123def456... - ``` - -2. **Kubernetes Format** - ``` - 11:cpu:/kubepods/pod123/def456... - ``` - -3. **containerd Format** - ``` - 10:cpu:/containerd/abc123... - ``` - -**Validation Rules:** -- Length must be 12 (short) or 64 (full) characters -- All characters must be hexadecimal - ---- - -### 4. ✅ eBPF Types Module - -**File:** `src/collectors/ebpf/types.rs` - -#### EbpfSyscallEvent Structure -```rust -#[repr(C)] -pub struct EbpfSyscallEvent { - pub pid: u32, - pub uid: u32, - pub syscall_id: u32, - pub _pad: u32, - pub timestamp: u64, - pub comm: [u8; 16], - pub data: EbpfEventData, -} -``` - -#### EbpfEventData Union -```rust -#[repr(C)] -pub union EbpfEventData { - pub execve: ExecveData, - pub connect: ConnectData, - pub openat: OpenatData, - pub ptrace: PtraceData, - pub raw: [u8; 128], -} -``` - -**Syscall-Specific Data:** - -**ExecveData:** -- `filename_len: u32` -- `filename: [u8; 128]` -- `argc: u32` - -**ConnectData:** -- `dst_ip: [u8; 16]` (IPv4 or IPv6) -- `dst_port: u16` -- `family: u16` (AF_INET or AF_INET6) - -**OpenatData:** -- `path_len: u32` -- `path: [u8; 256]` -- `flags: u32` - -**PtraceData:** -- `target_pid: u32` -- `request: u32` -- `addr: u64` -- `data: u64` - -**Conversion Functions:** -- `to_syscall_event()` - Convert eBPF event to userspace SyscallEvent -- `comm_str()` - Get command name as string -- `set_comm()` - Set command name - ---- - -### 5. ✅ Updated SyscallMonitor - -**File:** `src/collectors/ebpf/syscall_monitor.rs` - -**New Features:** -- Integrated `EventEnricher` for automatic enrichment -- Integrated `ContainerDetector` for container detection -- Uses `EventRingBuffer` for efficient buffering -- `current_container_id()` - Get current container -- `detect_container_for_pid(pid: u32)` - Detect container for PID -- `event_count()` - Get buffered event count -- `clear_events()` - Clear event buffer - ---- - -## Module Structure - -``` -src/collectors/ebpf/ -├── mod.rs ✅ Updated exports -├── loader.rs ✅ From TASK-003 -├── kernel.rs ✅ From TASK-003 -├── syscall_monitor.rs ✅ Updated with enrichment -├── programs.rs ✅ From TASK-003 -├── ring_buffer.rs ✅ From TASK-003 -├── enrichment.rs ✅ NEW -├── container.rs ✅ NEW -└── types.rs ✅ NEW -``` - ---- - -## Test Coverage - -### Tests Created: 25+ - -| Test File | Tests | Status | -|-----------|-------|--------| -| `execve_capture_test.rs` | 5 | ✅ Complete | -| `connect_capture_test.rs` | 4 | ✅ Complete | -| `openat_capture_test.rs` | 4 | ✅ Complete | -| `ptrace_capture_test.rs` | 3 | ✅ Complete | -| `event_enrichment_test.rs` | 13 | ✅ Complete | -| **Module Tests** | 15+ | ✅ Complete | -| **Total** | **40+** | | - -### Test Categories - -| Category | Tests | -|----------|-------| -| Syscall Capture | 16 | -| Enrichment | 13 | -| Container Detection | 8 | -| Types | 5 | - ---- - -## Code Quality - -### Cross-Platform Support -- ✅ All modules handle non-Linux gracefully -- ✅ Feature-gated compilation -- ✅ Clear error messages - -### Performance -- ✅ Caching for process info (EventEnricher) -- ✅ Caching for container IDs (ContainerDetector) -- ✅ Efficient ring buffer usage - -### Security -- ✅ Container ID validation -- ✅ Safe parsing of /proc files -- ✅ No unsafe code in userspace - ---- - -## Integration Points - -### With Event System -```rust -use stackdog::collectors::SyscallMonitor; - -let mut monitor = SyscallMonitor::new()?; -monitor.start()?; - -// Events are automatically enriched -let events = monitor.poll_events(); -for event in events { - // event.comm is populated - // event.container_id can be detected -} -``` - -### With Container Detection -```rust -use stackdog::collectors::ContainerDetector; - -let mut detector = ContainerDetector::new()?; - -// Detect container for current process -if let Some(container_id) = detector.current_container() { - println!("Running in container: {}", container_id); -} - -// Detect container for specific PID -if let Some(container_id) = detector.detect_container(1234) { - println!("PID 1234 is in container: {}", container_id); -} -``` - -### With Enrichment -```rust -use stackdog::collectors::EventEnricher; - -let mut enricher = EventEnricher::new()?; -let mut event = SyscallEvent::new(...); - -enricher.enrich(&mut event)?; - -// Now event has: -// - comm (process name) -// - Additional context -``` - ---- - -## Dependencies - -### Used -- `anyhow = "1"` - Error handling -- `log = "0.4"` - Logging -- `chrono = "0.4"` - Timestamps -- `thiserror = "1"` - Error types - -### No New Dependencies -All functionality implemented with existing dependencies. - ---- - -## Known Limitations - -### Current State -1. **eBPF Programs**: Still stubs - actual eBPF code needs TASK-004 completion -2. **Ring Buffer**: Uses Vec, not actual eBPF perf buffer -3. **Container Detection**: Only works with Docker/Kubernetes/containerd -4. **Process Cache**: No invalidation mechanism (stale data possible) - -### Next Steps -1. Implement actual eBPF programs in `ebpf/src/` -2. Connect ring buffer to eBPF perf buffer -3. Add cache invalidation with TTL -4. Add support for more container runtimes (Podman, LXC) - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| eBPF programs compile successfully | ⏳ eBPF code pending | -| Programs load and attach to kernel | ⏳ eBPF code pending | -| execve events captured on process spawn | ✅ Infrastructure ready | -| connect events captured on network connections | ✅ Infrastructure ready | -| openat events captured on file access | ✅ Infrastructure ready | -| ptrace events captured on debugging attempts | ✅ Infrastructure ready | -| Events enriched with container ID | ✅ Complete | -| All tests passing (target: 20+ tests) | ✅ 40+ tests | -| Documentation complete | ✅ Complete | - ---- - -## Files Modified/Created - -### Created (5 files) -- `src/collectors/ebpf/enrichment.rs` - Event enrichment -- `src/collectors/ebpf/container.rs` - Container detection -- `src/collectors/ebpf/types.rs` - eBPF types -- `tests/collectors/execve_capture_test.rs` - execve tests -- `tests/collectors/connect_capture_test.rs` - connect tests -- `tests/collectors/openat_capture_test.rs` - openat tests -- `tests/collectors/ptrace_capture_test.rs` - ptrace tests -- `tests/collectors/event_enrichment_test.rs` - enrichment tests - -### Modified -- `src/collectors/ebpf/mod.rs` - Added exports -- `src/collectors/ebpf/syscall_monitor.rs` - Added enrichment -- `tests/collectors/mod.rs` - Added test modules - ---- - -## Usage Example - -```rust -use stackdog::collectors::{SyscallMonitor, ContainerDetector}; - -// Create monitor with enrichment -let mut monitor = SyscallMonitor::new()?; -monitor.start()?; - -// Check if running in container -if let Some(container_id) = monitor.current_container_id() { - println!("Running in container: {}", container_id); -} - -// Poll for enriched events -loop { - let events = monitor.poll_events(); - for event in events { - println!( - "Syscall: {:?} | PID: {} | Command: {} | Container: {:?}", - event.syscall_type, - event.pid, - event.comm.as_ref().unwrap_or(&"unknown".to_string()), - monitor.detect_container_for_pid(event.pid) - ); - } - std::thread::sleep(std::time::Duration::from_millis(100)); -} -``` - ---- - -## Total Project Stats After TASK-004 - -| Metric | Count | -|--------|-------| -| **Total Tests** | 177+ | -| **Files Created** | 68+ | -| **Lines of Code** | 6500+ | -| **Documentation** | 14 files | - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-004.md b/docs/tasks/TASK-004.md deleted file mode 100644 index 8534b2a..0000000 --- a/docs/tasks/TASK-004.md +++ /dev/null @@ -1,203 +0,0 @@ -# Task Specification: TASK-004 - -## Implement Syscall Event Capture - -**Phase:** 1 - Foundation & eBPF Collectors -**Priority:** High -**Estimated Effort:** 3-4 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Implement actual eBPF programs for syscall monitoring and connect them to the userspace event capture system. This task transforms the stub implementation from TASK-003 into a working syscall monitoring system. - ---- - -## Requirements - -### 1. eBPF Programs (ebpf/src/) - -Implement eBPF tracepoint programs for: - -#### sys_enter_execve -- Capture process execution -- Extract: pid, uid, filename, arguments -- Send event to userspace via ring buffer - -#### sys_enter_connect -- Capture network connections -- Extract: pid, uid, destination IP, destination port -- Send event to userspace - -#### sys_enter_openat -- Capture file access -- Extract: pid, uid, file path, flags -- Send event to userspace - -#### sys_enter_ptrace -- Capture debugging attempts -- Extract: pid, uid, target pid, request type -- Send event to userspace - -### 2. Event Structure (Shared) - -Define C-compatible event structures for eBPF ↔ userspace communication: - -```c -struct SyscallEvent { - u32 pid; - u32 uid; - u64 timestamp; - u32 syscall_id; - char comm[16]; - // Union for syscall-specific data -}; -``` - -### 3. Ring Buffer Integration - -- Connect eBPF perf buffer to userspace -- Implement event polling loop -- Handle event deserialization -- Manage event loss - -### 4. Event Enrichment - -- Add container ID detection -- Add process tree information -- Add timestamp normalization - ---- - -## TDD Tests to Create - -### Test File: `tests/collectors/execve_capture_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_execve_event_captured_on_process_spawn() -#[test] -#[ignore = "requires root"] -fn test_execve_event_contains_filename() -#[test] -#[ignore = "requires root"] -fn test_execve_event_contains_pid() -#[test] -#[ignore = "requires root"] -fn test_execve_event_contains_uid() -``` - -### Test File: `tests/collectors/connect_capture_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_connect_event_captured_on_tcp_connection() -#[test] -#[ignore = "requires root"] -fn test_connect_event_contains_destination_ip() -#[test] -#[ignore = "requires root"] -fn test_connect_event_contains_destination_port() -``` - -### Test File: `tests/collectors/openat_capture_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_openat_event_captured_on_file_open() -#[test] -#[ignore = "requires root"] -fn test_openat_event_contains_file_path() -``` - -### Test File: `tests/collectors/ptrace_capture_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_ptrace_event_captured_on_trace_attempt() -``` - -### Test File: `tests/collectors/event_enrichment_test.rs` - -```rust -#[test] -fn test_container_id_detection() -#[test] -fn test_timestamp_normalization() -#[test] -fn test_process_tree_enrichment() -``` - ---- - -## Implementation Files - -### eBPF Programs (`ebpf/src/`) - -``` -ebpf/ -├── src/ -│ ├── lib.rs -│ ├── syscalls/ -│ │ ├── mod.rs -│ │ ├── execve.rs -│ │ ├── connect.rs -│ │ ├── openat.rs -│ │ └── ptrace.rs -│ ├── maps.rs -│ └── types.rs -``` - -### Userspace (`src/collectors/ebpf/`) - -``` -src/collectors/ebpf/ -├── mod.rs -├── loader.rs (from TASK-003) -├── event_reader.rs (NEW - event polling) -├── enrichment.rs (NEW - event enrichment) -└── container.rs (NEW - container detection) -``` - ---- - -## Acceptance Criteria - -- [ ] eBPF programs compile successfully -- [ ] Programs load and attach to kernel -- [ ] execve events captured on process spawn -- [ ] connect events captured on network connections -- [ ] openat events captured on file access -- [ ] ptrace events captured on debugging attempts -- [ ] Events enriched with container ID -- [ ] All tests passing (target: 20+ tests) -- [ ] Documentation complete - ---- - -## Dependencies - -- `aya = "0.12"` - eBPF framework -- `libc` - System calls -- `bollard` - Docker API (for container detection) - ---- - -## Risks - -| Risk | Impact | Probability | Mitigation | -|------|--------|-------------|------------| -| eBPF program rejection | High | Medium | Test on multiple kernels | -| Performance overhead | Medium | Low | Benchmark early | -| Container detection fails | Medium | Medium | Fallback to cgroup parsing | -| Event loss under load | High | Medium | Tune ring buffer size | - ---- - -*Created: 2026-03-13* diff --git a/docs/tasks/TASK-005-SUMMARY.md b/docs/tasks/TASK-005-SUMMARY.md deleted file mode 100644 index 460cfff..0000000 --- a/docs/tasks/TASK-005-SUMMARY.md +++ /dev/null @@ -1,406 +0,0 @@ -# TASK-005 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Rule Trait and Infrastructure - -**File:** `src/rules/rule.rs` - -#### RuleResult Enum -```rust -pub enum RuleResult { - Match, - NoMatch, - Error(String), -} -``` - -**Methods:** -- `is_match()` - Check if matched -- `is_no_match()` - Check if no match -- `is_error()` - Check if error -- `Display` trait implementation - -#### Rule Trait -```rust -pub trait Rule: Send + Sync { - fn evaluate(&self, event: &SecurityEvent) -> RuleResult; - fn name(&self) -> &str; - fn priority(&self) -> u32 { 100 } - fn enabled(&self) -> bool { true } -} -``` - ---- - -### 2. ✅ Rule Engine - -**File:** `src/rules/engine.rs` - -#### RuleEngine Struct -```rust -pub struct RuleEngine { - rules: Vec>, - enabled_rules: HashSet, -} -``` - -**Methods:** -- `new() -> Self` - Create engine -- `register_rule(rule: Box)` - Add rule -- `remove_rule(name: &str)` - Remove rule -- `evaluate(event: &SecurityEvent) -> Vec` - Evaluate all rules -- `evaluate_detailed(event: &SecurityEvent) -> Vec` - Detailed results -- `rule_count() -> usize` - Get count -- `clear_all_rules()` - Clear all -- `enable_rule(name: &str)` - Enable rule -- `disable_rule(name: &str)` - Disable rule -- `is_rule_enabled(name: &str) -> bool` - Check status -- `rule_names() -> Vec<&str>` - Get all names - -**Features:** -- Priority-based ordering (lower = higher priority) -- Enable/disable toggle -- Detailed evaluation results -- Rule removal by name - ---- - -### 3. ✅ Signature Database - -**File:** `src/rules/signatures.rs` - -#### ThreatCategory Enum -```rust -pub enum ThreatCategory { - Suspicious, - CryptoMiner, - ContainerEscape, - NetworkScanner, - PrivilegeEscalation, - DataExfiltration, - Malware, -} -``` - -#### Signature Struct -```rust -pub struct Signature { - name: String, - description: String, - severity: u8, - category: ThreatCategory, - syscall_patterns: Vec, -} -``` - -**Methods:** -- `new()` - Create signature -- `name()` - Get name -- `description()` - Get description -- `severity()` - Get severity (0-100) -- `category()` - Get category -- `matches(syscall_type: &SyscallType) -> bool` - Check match - -#### SignatureDatabase - -**Built-in Signatures (10):** - -| Name | Category | Severity | Patterns | -|------|----------|----------|----------| -| crypto_miner_execve | CryptoMiner | 70 | Execve, Setuid | -| container_escape_ptrace | ContainerEscape | 95 | Ptrace | -| container_escape_mount | ContainerEscape | 90 | Mount | -| network_scanner_connect | NetworkScanner | 60 | Connect | -| network_scanner_bind | NetworkScanner | 50 | Bind | -| privilege_escalation_setuid | PrivilegeEscalation | 85 | Setuid, Setgid | -| data_exfiltration_network | DataExfiltration | 75 | Connect, Sendto | -| malware_execve_tmp | Malware | 80 | Execve | -| suspicious_execveat | Suspicious | 50 | Execveat | -| suspicious_openat | Suspicious | 40 | Openat | - -**Methods:** -- `new() -> Self` - Create with built-in signatures -- `signature_count() -> usize` - Get count -- `add_signature(signature: Signature)` - Add custom -- `remove_signature(name: &str)` - Remove by name -- `get_signatures_by_category(category: &ThreatCategory) -> Vec<&Signature>` - Filter by category -- `find_matching(syscall_type: &SyscallType) -> Vec<&Signature>` - Find matches -- `detect(event: &SecurityEvent) -> Vec<&Signature>` - Detect threats in event - ---- - -### 4. ✅ Built-in Rules - -**File:** `src/rules/builtin.rs` - -#### SyscallAllowlistRule -- Matches if syscall is in allowed list -- Priority: 50 - -#### SyscallBlocklistRule -- Matches if syscall is in blocked list (violation) -- Priority: 10 (high priority for security) - -#### ProcessExecutionRule -- Matches Execve, Execveat syscalls -- Priority: 30 - -#### NetworkConnectionRule -- Matches Connect, Accept, Bind, Listen, Socket -- Priority: 40 - -#### FileAccessRule -- Matches Open, Openat, Close, Read, Write -- Priority: 60 - ---- - -### 5. ✅ Rule Results - -**File:** `src/rules/result.rs` - -#### Severity Enum -```rust -pub enum Severity { - Info = 0, - Low = 20, - Medium = 40, - High = 70, - Critical = 90, -} -``` - -**Methods:** -- `from_score(score: u8) -> Self` - Convert score to severity -- `score() -> u8` - Get numeric score -- `Display` trait implementation -- `PartialOrd` for comparison - -#### RuleEvaluationResult Struct -```rust -pub struct RuleEvaluationResult { - rule_name: String, - event: SecurityEvent, - result: RuleResult, - timestamp: DateTime, -} -``` - -**Methods:** -- `new(rule_name, event, result) -> Self` -- `rule_name() -> &str` -- `event() -> &SecurityEvent` -- `result() -> &RuleResult` -- `timestamp() -> DateTime` -- `matched() -> bool` -- `not_matched() -> bool` -- `has_error() -> bool` - -#### Utility Functions -- `calculate_aggregate_severity(severities: &[Severity]) -> Severity` - Get highest -- `calculate_severity_from_results(results: &[RuleEvaluationResult], base: &[Severity]) -> Severity` - ---- - -## Test Coverage - -### Tests Created: 35+ - -| Test File | Tests | Status | -|-----------|-------|--------| -| `rule_engine_test.rs` | 10 | ✅ Complete | -| `signature_test.rs` | 14 | ✅ Complete | -| `builtin_rules_test.rs` | 17 | ✅ Complete | -| `rule_result_test.rs` | 13 | ✅ Complete | -| **Module Tests** | 5+ | ✅ Complete | -| **Total** | **59+** | | - -### Test Coverage by Category - -| Category | Tests | -|----------|-------| -| Rule Engine | 10 | -| Signatures | 14 | -| Built-in Rules | 17 | -| Rule Results | 13 | -| Module Tests | 5 | - ---- - -## Module Structure - -``` -src/rules/ -├── mod.rs ✅ Updated exports -├── engine.rs ✅ Rule engine -├── rule.rs ✅ Rule trait -├── signatures.rs ✅ Signature database -├── builtin.rs ✅ Built-in rules -└── result.rs ✅ Result types -``` - ---- - -## Code Quality - -### Design Patterns -- **Trait-based polymorphism** - Rule trait for extensibility -- **Strategy pattern** - Different rule implementations -- **Builder pattern** - Signature construction -- **Priority ordering** - Rules sorted by priority - -### Error Handling -- `RuleResult::Error` for evaluation errors -- `anyhow::Result` for fallible operations -- Graceful handling of unknown events - -### Performance -- Priority-based sorting for efficient evaluation -- HashSet for O(1) enable/disable checks -- Vec for rule storage (fast iteration) - ---- - -## Integration Points - -### With Event System -```rust -use stackdog::rules::{RuleEngine, SignatureDatabase}; -use stackdog::events::security::SecurityEvent; - -let mut engine = RuleEngine::new(); -let db = SignatureDatabase::new(); - -// Add signature-based rule -engine.register_rule(Box::new(SignatureRule::new(db))); - -// Evaluate events -let events = monitor.poll_events(); -for event in events { - let results = engine.evaluate(&event); - for result in results { - if result.is_match() { - println!("Rule matched!"); - } - } -} -``` - -### With Alerting (Future) -```rust -let detailed_results = engine.evaluate_detailed(&event); -for result in detailed_results { - if result.matched() { - alerting::create_alert( - result.rule_name(), - calculate_severity(&result), - result.event(), - ); - } -} -``` - ---- - -## Usage Example - -```rust -use stackdog::rules::{RuleEngine, SignatureDatabase, ThreatCategory}; -use stackdog::rules::builtin::{ - SyscallBlocklistRule, ProcessExecutionRule, -}; -use stackdog::events::syscall::SyscallType; - -// Create engine -let mut engine = RuleEngine::new(); - -// Add built-in rules -engine.register_rule(Box::new(SyscallBlocklistRule::new( - vec![SyscallType::Ptrace, SyscallType::Setuid] -))); - -engine.register_rule(Box::new(ProcessExecutionRule::new())); - -// Get signature database -let db = SignatureDatabase::new(); -println!("Loaded {} signatures", db.signature_count()); - -// Evaluate event -let event = SecurityEvent::Syscall(SyscallEvent::new( - 1234, 1000, SyscallType::Ptrace, Utc::now(), -)); - -let results = engine.evaluate(&event); -let matches = results.iter() - .filter(|r| r.is_match()) - .count(); - -println!("{} rules matched", matches); - -// Get matching signatures -let sig_matches = db.detect(&event); -for sig in sig_matches { - println!( - "Threat detected: {} (Severity: {}, Category: {})", - sig.name(), - sig.severity(), - sig.category() - ); -} -``` - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| Rule trait fully implemented | ✅ Complete | -| Rule engine with priority ordering | ✅ Complete | -| 10+ built-in signatures | ✅ 10 signatures | -| 5+ built-in rules | ✅ 5 rules | -| Rule DSL parsing | ⏳ Deferred to TASK-006 | -| All tests passing (target: 30+ tests) | ✅ 59+ tests | -| Documentation complete | ✅ Complete | - ---- - -## Files Modified/Created - -### Created (5 files) -- `src/rules/engine.rs` - Rule engine -- `src/rules/rule.rs` - Rule trait (enhanced) -- `src/rules/signatures.rs` - Signature database (enhanced) -- `src/rules/builtin.rs` - Built-in rules (NEW) -- `src/rules/result.rs` - Result types (NEW) -- `tests/rules/rule_engine_test.rs` - Engine tests -- `tests/rules/signature_test.rs` - Signature tests -- `tests/rules/builtin_rules_test.rs` - Built-in rule tests -- `tests/rules/rule_result_test.rs` - Result tests - -### Modified -- `src/rules/mod.rs` - Updated exports -- `src/events/syscall.rs` - Added new SyscallType variants -- `tests/rules/mod.rs` - Added test modules - ---- - -## Total Project Stats After TASK-005 - -| Metric | Count | -|--------|-------| -| **Total Tests** | 236+ | -| **Files Created** | 73+ | -| **Lines of Code** | 8000+ | -| **Documentation** | 16 files | - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-005.md b/docs/tasks/TASK-005.md deleted file mode 100644 index 8930131..0000000 --- a/docs/tasks/TASK-005.md +++ /dev/null @@ -1,165 +0,0 @@ -# Task Specification: TASK-005 - -## Create Rule Engine Infrastructure - -**Phase:** 1 - Foundation & eBPF Collectors -**Priority:** High -**Estimated Effort:** 2-3 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Implement a flexible rule engine for security event evaluation. The rule engine will support signature-based detection, pattern matching, and configurable rules with priority-based evaluation. - ---- - -## Requirements - -### 1. Rule Trait and Implementations - -Define a `Rule` trait with: -- `evaluate()` - Evaluate rule against event -- `name()` - Rule identifier -- `priority()` - Evaluation priority -- `enabled()` - Rule status - -Implement built-in rules: -- Syscall allowlist/blocklist -- Process execution rules -- Network connection rules -- File access rules - -### 2. Rule Engine - -Implement `RuleEngine` with: -- Rule registration and management -- Priority-based evaluation order -- Rule chaining -- Result aggregation -- Performance metrics - -### 3. Signature Database - -Implement threat signature database: -- Known threat patterns -- Crypto miner signatures -- Container escape signatures -- Network scanner signatures -- Signature matching engine - -### 4. Rule DSL (Domain Specific Language) - -Create simple rule definition language: -```yaml -rule: suspicious_execve -description: Detect execution in temp directories -priority: 80 -condition: - syscall: execve - path_matches: ["/tmp/*", "/var/tmp/*"] -action: alert -severity: high -``` - ---- - -## TDD Tests to Create - -### Test File: `tests/rules/rule_engine_test.rs` - -```rust -#[test] -fn test_rule_engine_creation() -#[test] -fn test_rule_registration() -#[test] -fn test_rule_priority_ordering() -#[test] -fn test_rule_evaluation_single() -#[test] -fn test_rule_evaluation_multiple() -#[test] -fn test_rule_removal() -#[test] -fn test_rule_enable_disable() -``` - -### Test File: `tests/rules/signature_test.rs` - -```rust -#[test] -fn test_signature_creation() -#[test] -fn test_signature_matching() -#[test] -fn test_builtin_signatures() -#[test] -fn test_crypto_miner_signature() -#[test] -fn test_container_escape_signature() -#[test] -fn test_network_scanner_signature() -``` - -### Test File: `tests/rules/builtin_rules_test.rs` - -```rust -#[test] -fn test_syscall_allowlist_rule() -#[test] -fn test_syscall_blocklist_rule() -#[test] -fn test_process_execution_rule() -#[test] -fn test_network_connection_rule() -#[test] -fn test_file_access_rule() -``` - -### Test File: `tests/rules/rule_result_test.rs` - -```rust -#[test] -fn test_rule_result_match() -#[test] -fn test_rule_result_no_match() -#[test] -fn test_rule_result_aggregation() -#[test] -fn test_severity_calculation() -``` - ---- - -## Implementation Files - -### Rule Engine (`src/rules/`) - -``` -src/rules/ -├── mod.rs -├── engine.rs (from TASK-001, enhance) -├── rule.rs (from TASK-001, enhance) -├── signatures.rs (from TASK-001, enhance) -├── builtin.rs (NEW - built-in rules) -├── dsl.rs (NEW - rule DSL) -└── result.rs (NEW - rule results) -``` - ---- - -## Acceptance Criteria - -- [ ] Rule trait fully implemented -- [ ] Rule engine with priority ordering -- [ ] 10+ built-in signatures -- [ ] 5+ built-in rules -- [ ] Rule DSL parsing -- [ ] All tests passing (target: 30+ tests) -- [ ] Documentation complete - ---- - -*Created: 2026-03-13* diff --git a/docs/tasks/TASK-006-SUMMARY.md b/docs/tasks/TASK-006-SUMMARY.md deleted file mode 100644 index ebbf730..0000000 --- a/docs/tasks/TASK-006-SUMMARY.md +++ /dev/null @@ -1,395 +0,0 @@ -# TASK-006 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Advanced Signature Matching - -**File:** `src/rules/signature_matcher.rs` - -#### PatternMatch Struct -```rust -pub struct PatternMatch { - syscalls: Vec, - time_window: Option, - description: String, -} -``` - -**Builder Methods:** -- `with_syscall(SyscallType)` - Add syscall to pattern -- `then_syscall(SyscallType)` - Add next in sequence -- `within_seconds(u64)` - Set time window -- `with_description(String)` - Set description - -#### MatchResult Struct -```rust -pub struct MatchResult { - matches: Vec, - is_match: bool, - confidence: f64, -} -``` - -**Methods:** -- `matches()` - Get matched signatures -- `is_match()` - Check if matched -- `confidence()` - Get confidence score (0.0-1.0) - -#### SignatureMatcher Struct -```rust -pub struct SignatureMatcher { - db: SignatureDatabase, - patterns: Vec, -} -``` - -**Methods:** -- `new() -> Self` - Create matcher -- `add_pattern(pattern: PatternMatch)` - Add pattern -- `match_single(event: &SecurityEvent) -> MatchResult` - Single event matching -- `match_sequence(events: &[SecurityEvent]) -> MatchResult` - Multi-event matching -- `database() -> &SignatureDatabase` - Get database -- `patterns() -> &[PatternMatch]` - Get patterns - -**Features:** -- Single event signature matching -- Multi-event pattern matching -- Temporal correlation (time window) -- Sequence detection (ordered patterns) -- Confidence scoring - ---- - -### 2. ✅ Threat Scoring Engine - -**File:** `src/rules/threat_scorer.rs` - -#### ThreatScore Struct -```rust -pub struct ThreatScore { - value: u8, // 0-100 -} -``` - -**Methods:** -- `new(value: u8) -> Self` - Create score -- `value() -> u8` - Get value -- `severity() -> Severity` - Convert to severity -- `exceeds_threshold(threshold: u8) -> bool` - Check threshold -- `is_high_or_higher() -> bool` - Check if >= 70 -- `is_critical() -> bool` - Check if >= 90 -- `add(&mut self, value: u8)` - Add to score (capped at 100) - -#### ScoringConfig Struct -```rust -pub struct ScoringConfig { - base_score: u8, - multiplier: f64, - time_decay_enabled: bool, - decay_half_life_seconds: u64, -} -``` - -**Builder Methods:** -- `with_base_score(u8)` - Set base score -- `with_multiplier(f64)` - Set multiplier -- `with_time_decay(bool)` - Enable/disable decay -- `with_decay_half_life(u64)` - Set half-life - -#### ThreatScorer Struct -```rust -pub struct ThreatScorer { - config: ScoringConfig, - matcher: SignatureMatcher, -} -``` - -**Methods:** -- `new() -> Self` - Create with default config -- `with_config(config: ScoringConfig) -> Self` - Custom config -- `with_matcher(matcher: SignatureMatcher) -> Self` - Custom matcher -- `calculate_score(event: &SecurityEvent) -> ThreatScore` - Single event score -- `calculate_cumulative_score(events: &[SecurityEvent]) -> ThreatScore` - Multi-event score - -**Features:** -- Base score configuration -- Multiplier support -- Time decay (ready for implementation) -- Cumulative scoring with bonus for multiple events - -#### Utility Functions -- `aggregate_severities(severities: &[Severity]) -> Severity` - Get highest -- `calculate_severity_from_scores(scores: &[ThreatScore]) -> Severity` - From scores - ---- - -### 3. ✅ Detection Statistics - -**File:** `src/rules/stats.rs` - -#### DetectionStats Struct -```rust -pub struct DetectionStats { - events_processed: u64, - signatures_matched: u64, - false_positives: u64, - true_positives: u64, - start_time: DateTime, - last_updated: DateTime, -} -``` - -**Methods:** -- `new() -> Self` - Create stats -- `record_event()` - Record event processed -- `record_match()` - Record signature match -- `record_false_positive()` - Record false positive -- `events_processed() -> u64` - Get count -- `signatures_matched() -> u64` - Get count -- `detection_rate() -> f64` - Calculate rate (matches/events) -- `false_positive_rate() -> f64` - Calculate FP rate -- `precision() -> f64` - Calculate precision -- `uptime() -> Duration` - Get uptime -- `events_per_second() -> f64` - Calculate throughput - -#### StatsTracker Struct -```rust -pub struct StatsTracker { - stats: DetectionStats, -} -``` - -**Methods:** -- `new() -> Result` - Create tracker -- `record_event(event: &SecurityEvent, matched: bool)` - Record with result -- `stats() -> &DetectionStats` - Get stats -- `stats_mut() -> &mut DetectionStats` - Get mutable stats -- `reset()` - Reset all stats - -**Features:** -- Real-time tracking -- Detection rate calculation -- False positive tracking -- Precision metrics -- Throughput monitoring - ---- - -## Test Coverage - -### Tests Created: 35+ - -| Test File | Tests | Status | -|-----------|-------|--------| -| `signature_matching_test.rs` | 10 | ✅ Complete | -| `threat_scoring_test.rs` | 13 | ✅ Complete | -| `detection_stats_test.rs` | 13 | ✅ Complete | -| **Module Tests** | 5+ | ✅ Complete | -| **Total** | **41+** | | - -### Test Coverage by Category - -| Category | Tests | -|----------|-------| -| Signature Matching | 10 | -| Threat Scoring | 13 | -| Detection Statistics | 13 | -| Module Tests | 5 | - ---- - -## Module Structure - -``` -src/rules/ -├── mod.rs ✅ Updated exports -├── engine.rs ✅ From TASK-005 -├── rule.rs ✅ From TASK-005 -├── signatures.rs ✅ From TASK-005 -├── builtin.rs ✅ From TASK-005 -├── result.rs ✅ From TASK-005 -├── signature_matcher.rs ✅ NEW -├── threat_scorer.rs ✅ NEW -└── stats.rs ✅ NEW -``` - ---- - -## Code Quality - -### Design Patterns -- **Builder Pattern** - PatternMatch, ScoringConfig -- **Strategy Pattern** - Different scoring strategies -- **Aggregate Pattern** - Severity aggregation -- **Observer Pattern** - Stats tracking - -### Performance -- Efficient pattern matching algorithm -- O(n) sequence matching -- Configurable time-decay scoring -- Real-time statistics tracking - -### Error Handling -- Graceful handling of empty event sequences -- Safe division (zero checks) -- Result types for match outcomes - ---- - -## Integration Points - -### With Event System -```rust -use stackdog::rules::{SignatureMatcher, ThreatScorer, StatsTracker}; - -let mut matcher = SignatureMatcher::new(); -let mut scorer = ThreatScorer::new(); -let mut tracker = StatsTracker::new()?; - -// Add pattern -matcher.add_pattern( - PatternMatch::new() - .with_syscall(SyscallType::Execve) - .then_syscall(SyscallType::Connect) - .within_seconds(60) -); - -// Process events -for event in events { - let match_result = matcher.match_single(&event); - let score = scorer.calculate_score(&event); - - tracker.record_event(&event, match_result.is_match()); - - if score.is_high_or_higher() { - // Generate alert - } -} -``` - -### With Alerting (Future) -```rust -let stats = tracker.stats(); -if stats.detection_rate() > 0.5 { - // High detection rate - possible attack - alerting::create_alert( - "High detection rate", - Severity::High, - format!("Detection rate: {:.1}%", stats.detection_rate() * 100.0), - ); -} -``` - ---- - -## Usage Example - -```rust -use stackdog::rules::{ - SignatureMatcher, ThreatScorer, StatsTracker, - PatternMatch, ScoringConfig, -}; -use stackdog::events::syscall::SyscallType; -use stackdog::events::security::SecurityEvent; - -// Create matcher with pattern -let mut matcher = SignatureMatcher::new(); -matcher.add_pattern( - PatternMatch::new() - .with_syscall(SyscallType::Execve) - .then_syscall(SyscallType::Ptrace) - .within_seconds(300) - .with_description("Suspicious process debugging") -); - -// Create scorer with custom config -let config = ScoringConfig::default() - .with_base_score(60) - .with_multiplier(1.2); -let scorer = ThreatScorer::with_config(config); - -// Create stats tracker -let mut tracker = StatsTracker::new()?; - -// Process events -let events = vec![ - SecurityEvent::Syscall(SyscallEvent::new(1234, 1000, SyscallType::Execve, Utc::now())), - SecurityEvent::Syscall(SyscallEvent::new(1234, 1000, SyscallType::Ptrace, Utc::now())), -]; - -// Check for pattern match -let pattern_result = matcher.match_sequence(&events); -if pattern_result.is_match() { - println!("Pattern matched: {}", pattern_result); -} - -// Calculate scores -for event in &events { - let score = scorer.calculate_score(event); - tracker.record_event(event, score.value() > 0); - - if score.is_high_or_higher() { - println!("High threat score: {}", score.value()); - } -} - -// Get statistics -let stats = tracker.stats(); -println!( - "Processed {} events, {} matches, rate: {:.1}%", - stats.events_processed(), - stats.signatures_matched(), - stats.detection_rate() * 100.0 -); -``` - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| Multi-event pattern matching implemented | ✅ Complete | -| Temporal correlation working | ✅ Complete | -| Threat scoring with time decay | ✅ Complete (config ready) | -| Signature DSL parsing | ⏳ Deferred to TASK-007 | -| Detection statistics tracking | ✅ Complete | -| All tests passing (target: 25+ tests) | ✅ 41+ tests | -| Documentation complete | ✅ Complete | - ---- - -## Files Modified/Created - -### Created (3 files) -- `src/rules/signature_matcher.rs` - Advanced matching -- `src/rules/threat_scorer.rs` - Scoring engine -- `src/rules/stats.rs` - Detection statistics -- `tests/rules/signature_matching_test.rs` - Matching tests -- `tests/rules/threat_scoring_test.rs` - Scoring tests -- `tests/rules/detection_stats_test.rs` - Stats tests - -### Modified -- `src/rules/mod.rs` - Updated exports -- `tests/rules/mod.rs` - Added test modules - ---- - -## Total Project Stats After TASK-006 - -| Metric | Count | -|--------|-------| -| **Total Tests** | 277+ | -| **Files Created** | 76+ | -| **Lines of Code** | 9000+ | -| **Documentation** | 18 files | - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-006.md b/docs/tasks/TASK-006.md deleted file mode 100644 index d5dbc6a..0000000 --- a/docs/tasks/TASK-006.md +++ /dev/null @@ -1,138 +0,0 @@ -# Task Specification: TASK-006 - -## Implement Signature-based Detection - -**Phase:** 2 - Detection & Response -**Priority:** High -**Estimated Effort:** 2-3 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Implement advanced signature-based detection capabilities including multi-event pattern matching, threat scoring, and signature rule definitions. This task builds on the rule engine from TASK-005 to provide comprehensive threat detection. - ---- - -## Requirements - -### 1. Advanced Signature Matching - -Implement signature matching engine with: -- Single event matching (from TASK-005) -- Multi-event pattern matching -- Temporal correlation (events within time window) -- Sequence detection (ordered event patterns) - -### 2. Threat Scoring Engine - -Implement threat scoring with: -- Base severity from signatures -- Cumulative scoring (multiple matches) -- Time-decay scoring (recent events weighted higher) -- Threshold-based alerting - -### 3. Signature Rule DSL - -Create YAML-based rule definition: -```yaml -rule: suspicious_process_chain -description: Detects suspicious process execution chain -severity: 80 -category: malware -patterns: - - syscall: execve - path: "/tmp/*" - - syscall: execve - path: "/var/tmp/*" - within_seconds: 60 -action: alert -``` - -### 4. Detection Statistics - -Track detection metrics: -- Events processed -- Signatures matched -- False positive tracking -- Detection rate - ---- - -## TDD Tests to Create - -### Test File: `tests/rules/signature_matching_test.rs` - -```rust -#[test] -fn test_single_event_signature_match() -#[test] -fn test_multi_event_pattern_match() -#[test] -fn test_temporal_correlation_match() -#[test] -fn test_sequence_detection() -#[test] -fn test_signature_match_with_no_temporal_match() -``` - -### Test File: `tests/rules/threat_scoring_test.rs` - -```rust -#[test] -fn test_threat_score_calculation() -#[test] -fn test_cumulative_scoring() -#[test] -fn test_time_decay_scoring() -#[test] -fn test_threshold_alerting() -#[test] -fn test_severity_aggregation() -``` - -### Test File: `tests/rules/detection_stats_test.rs` - -```rust -#[test] -fn test_detection_statistics_tracking() -#[test] -fn test_events_processed_count() -#[test] -fn test_signatures_matched_count() -#[test] -fn test_detection_rate_calculation() -``` - ---- - -## Implementation Files - -### Detection Engine (`src/rules/`) - -``` -src/rules/ -├── mod.rs -├── engine.rs (from TASK-005, enhance) -├── signature_matcher.rs (NEW - advanced matching) -├── threat_scorer.rs (NEW - scoring engine) -├── dsl.rs (NEW - rule DSL) -└── stats.rs (NEW - detection statistics) -``` - ---- - -## Acceptance Criteria - -- [ ] Multi-event pattern matching implemented -- [ ] Temporal correlation working -- [ ] Threat scoring with time decay -- [ ] Signature DSL parsing -- [ ] Detection statistics tracking -- [ ] All tests passing (target: 25+ tests) -- [ ] Documentation complete - ---- - -*Created: 2026-03-13* diff --git a/docs/tasks/TASK-007-SUMMARY.md b/docs/tasks/TASK-007-SUMMARY.md deleted file mode 100644 index f1db630..0000000 --- a/docs/tasks/TASK-007-SUMMARY.md +++ /dev/null @@ -1,478 +0,0 @@ -# TASK-007 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Alert Data Model - -**File:** `src/alerting/alert.rs` - -#### AlertType Enum -```rust -pub enum AlertType { - ThreatDetected, - AnomalyDetected, - RuleViolation, - ThresholdExceeded, - QuarantineApplied, - SystemEvent, -} -``` - -#### AlertSeverity Enum -```rust -pub enum AlertSeverity { - Info = 0, - Low = 20, - Medium = 40, - High = 70, - Critical = 90, -} -``` - -#### AlertStatus Enum -```rust -pub enum AlertStatus { - New, - Acknowledged, - Resolved, - FalsePositive, -} -``` - -#### Alert Struct -```rust -pub struct Alert { - id: String, // UUID - alert_type: AlertType, - severity: AlertSeverity, - message: String, - status: AlertStatus, - timestamp: DateTime, - source_event: Option, - metadata: HashMap, - resolved_at: Option>, - resolution_note: Option, -} -``` - -**Methods:** -- `new(alert_type, severity, message) -> Self` -- `id() -> &str` -- `alert_type() -> AlertType` -- `severity() -> AlertSeverity` -- `message() -> &str` -- `status() -> AlertStatus` -- `timestamp() -> DateTime` -- `source_event() -> Option<&SecurityEvent>` -- `set_source_event(event)` -- `metadata() -> &HashMap` -- `add_metadata(key, value)` -- `acknowledge()` - Transition to Acknowledged -- `resolve()` - Transition to Resolved -- `set_resolution_note(note)` -- `fingerprint() -> String` - For deduplication - ---- - -### 2. ✅ Alert Manager - -**File:** `src/alerting/manager.rs` - -#### AlertStats Struct -```rust -pub struct AlertStats { - pub total_count: u64, - pub new_count: u64, - pub acknowledged_count: u64, - pub resolved_count: u64, - pub false_positive_count: u64, -} -``` - -#### AlertManager Struct -```rust -pub struct AlertManager { - alerts: Arc>>, - stats: Arc>, -} -``` - -**Methods:** -- `new() -> Result` -- `generate_alert(type, severity, message, source) -> Result` -- `get_alert(id: &str) -> Option` -- `get_all_alerts() -> Vec` -- `get_alerts_by_severity(severity) -> Vec` -- `get_alerts_by_status(status) -> Vec` -- `acknowledge_alert(id: &str) -> Result<()>` -- `resolve_alert(id: &str, note: String) -> Result<()>` -- `alert_count() -> usize` -- `get_stats() -> AlertStats` -- `clear_resolved_alerts() -> usize` - -**Features:** -- Thread-safe storage (Arc) -- Alert lifecycle management -- Statistics tracking -- Query by severity and status - ---- - -### 3. ✅ Alert Deduplication - -**File:** `src/alerting/dedup.rs` - -#### DedupConfig Struct -```rust -pub struct DedupConfig { - enabled: bool, - window_seconds: u64, - aggregation: bool, -} -``` - -**Builder Methods:** -- `with_enabled(bool)` -- `with_window_seconds(u64)` -- `with_aggregation(bool)` - -#### Fingerprint Struct -```rust -pub struct Fingerprint(String); -``` - -#### DedupResult Struct -```rust -pub struct DedupResult { - pub is_duplicate: bool, - pub count: u32, - pub first_seen: DateTime, -} -``` - -#### AlertDeduplicator Struct -```rust -pub struct AlertDeduplicator { - config: DedupConfig, - fingerprints: HashMap, - stats: DedupStats, -} -``` - -**Methods:** -- `new(config: DedupConfig) -> Self` -- `calculate_fingerprint(alert: &Alert) -> Fingerprint` -- `is_duplicate(alert: &Alert) -> bool` -- `check(alert: &Alert) -> DedupResult` -- `get_stats() -> DedupStatsPublic` -- `clear_expired()` - Remove old fingerprints - -**Features:** -- Time-window based deduplication -- Alert aggregation (count duplicates) -- Configurable window (default 5 minutes) -- Statistics tracking - ---- - -### 4. ✅ Notification Channels - -**File:** `src/alerting/notifications.rs` - -#### NotificationConfig Struct -```rust -pub struct NotificationConfig { - slack_webhook: Option, - smtp_host: Option, - smtp_port: Option, - webhook_url: Option, - email_recipients: Vec, -} -``` - -**Builder Methods:** -- `with_slack_webhook(url: String)` -- `with_smtp_host(host: String)` -- `with_smtp_port(port: u16)` -- `with_webhook_url(url: String)` - -#### NotificationChannel Enum -```rust -pub enum NotificationChannel { - Console, - Slack, - Email, - Webhook, -} -``` - -**Methods:** -- `send(alert: &Alert, config: &NotificationConfig) -> Result` - -#### NotificationResult Enum -```rust -pub enum NotificationResult { - Success(String), - Failure(String), -} -``` - -**Utility Functions:** -- `route_by_severity(severity) -> Vec` -- `severity_to_slack_color(severity) -> &'static str` -- `build_slack_message(alert: &Alert) -> String` -- `build_webhook_payload(alert: &Alert) -> String` - -**Features:** -- 4 notification channels -- Severity-based routing -- Slack message formatting -- Webhook payload building - ---- - -## Test Coverage - -### Tests Created: 35+ - -| Test File | Tests | Status | -|-----------|-------|--------| -| `alert_test.rs` | 14 | ✅ Complete | -| `alert_manager_test.rs` | 12 | ✅ Complete | -| `deduplication_test.rs` | 13 | ✅ Complete | -| `notifications_test.rs` | 8 | ✅ Complete | -| **Module Tests** | 5+ | ✅ Complete | -| **Total** | **52+** | | - -### Test Coverage by Category - -| Category | Tests | -|----------|-------| -| Alert Data Model | 14 | -| Alert Manager | 12 | -| Deduplication | 13 | -| Notifications | 8 | -| Module Tests | 5 | - ---- - -## Module Structure - -``` -src/alerting/ -├── mod.rs ✅ Updated exports -├── alert.rs ✅ Alert data model -├── manager.rs ✅ Alert management -├── dedup.rs ✅ Deduplication -└── notifications.rs ✅ Notification channels -``` - ---- - -## Code Quality - -### Design Patterns -- **Builder Pattern** - DedupConfig, NotificationConfig -- **Strategy Pattern** - Different notification channels -- **State Pattern** - Alert status transitions -- **Factory Pattern** - Alert generation - -### Thread Safety -- `Arc>` for shared state -- Safe concurrent access to alerts -- Lock-free reads where possible - -### Error Handling -- `anyhow::Result` for fallible operations -- Graceful handling of missing alerts -- Notification failure handling - ---- - -## Integration Points - -### With Rule Engine -```rust -use stackdog::alerting::AlertManager; -use stackdog::rules::RuleEngine; - -let mut alert_manager = AlertManager::new()?; -let mut rule_engine = RuleEngine::new(); - -// Evaluate rules -for event in events { - let results = rule_engine.evaluate(&event); - - for result in results { - if result.is_match() { - let _ = alert_manager.generate_alert( - AlertType::RuleViolation, - result.severity(), - format!("Rule matched: {}", result.rule_name()), - Some(event.clone()), - ); - } - } -} -``` - -### With Threat Scorer -```rust -use stackdog::rules::ThreatScorer; - -let scorer = ThreatScorer::new(); -let score = scorer.calculate_score(&event); - -if score.is_critical() { - let _ = alert_manager.generate_alert( - AlertType::ThreatDetected, - AlertSeverity::Critical, - format!("Critical threat score: {}", score.value()), - Some(event.clone()), - ); -} -``` - -### With Deduplication -```rust -use stackdog::alerting::AlertDeduplicator; - -let mut dedup = AlertDeduplicator::new(DedupConfig::default()); - -for alert in alerts { - let result = dedup.check(&alert); - - if result.is_duplicate { - log::info!("Duplicate alert (count: {})", result.count); - } else { - // Send notification - send_notification(&alert); - } -} -``` - ---- - -## Usage Example - -```rust -use stackdog::alerting::{ - AlertManager, AlertType, AlertSeverity, - AlertDeduplicator, DedupConfig, - NotificationChannel, NotificationConfig, -}; - -// Create alert manager -let mut alert_manager = AlertManager::new()?; - -// Create deduplicator -let dedup_config = DedupConfig::default() - .with_window_seconds(300) - .with_aggregation(true); -let mut dedup = AlertDeduplicator::new(dedup_config); - -// Generate alert -let alert = alert_manager.generate_alert( - AlertType::ThreatDetected, - AlertSeverity::High, - "Suspicious process execution detected".to_string(), - Some(event), -)?; - -// Check for duplicates -let dedup_result = dedup.check(&alert); - -if !dedup_result.is_duplicate { - // Send notifications - let config = NotificationConfig::default() - .with_slack_webhook("https://hooks.slack.com/...".to_string()); - - let channels = vec![ - NotificationChannel::Console, - NotificationChannel::Slack, - ]; - - for channel in channels { - let result = channel.send(&alert, &config); - match result { - NotificationResult::Success(msg) => log::info!("Sent: {}", msg), - NotificationResult::Failure(msg) => log::error!("Failed: {}", msg), - } - } -} - -// Acknowledge alert -let alert_id = alert.id().to_string(); -alert_manager.acknowledge_alert(&alert_id)?; - -// Later, resolve alert -alert_manager.resolve_alert( - &alert_id, - "Investigated and mitigated".to_string() -)?; - -// Get statistics -let stats = alert_manager.get_stats(); -println!( - "Total: {}, New: {}, Acknowledged: {}, Resolved: {}", - stats.total_count, - stats.new_count, - stats.acknowledged_count, - stats.resolved_count -); -``` - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| Alert data model implemented | ✅ Complete | -| Alert generation from rules working | ✅ Complete | -| Deduplication with time windows | ✅ Complete | -| 4 notification channels implemented | ✅ Complete | -| Alert storage and querying | ✅ Complete | -| Status management (new, ack, resolved) | ✅ Complete | -| All tests passing (target: 30+ tests) | ✅ 52+ tests | -| Documentation complete | ✅ Complete | - ---- - -## Files Modified/Created - -### Created (4 files) -- `src/alerting/alert.rs` - Alert data model -- `src/alerting/manager.rs` - Alert management -- `src/alerting/dedup.rs` - Deduplication -- `src/alerting/notifications.rs` - Notification channels -- `tests/alerting/alert_test.rs` - Alert tests -- `tests/alerting/alert_manager_test.rs` - Manager tests -- `tests/alerting/deduplication_test.rs` - Dedup tests -- `tests/alerting/notifications_test.rs` - Notification tests - -### Modified -- `src/alerting/mod.rs` - Updated exports -- `src/lib.rs` - Added alerting re-exports -- `tests/alerting/mod.rs` - Added test modules - ---- - -## Total Project Stats After TASK-007 - -| Metric | Count | -|--------|-------| -| **Total Tests** | 329+ | -| **Files Created** | 80+ | -| **Lines of Code** | 10000+ | -| **Documentation** | 20 files | - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-007.md b/docs/tasks/TASK-007.md deleted file mode 100644 index 34364ca..0000000 --- a/docs/tasks/TASK-007.md +++ /dev/null @@ -1,166 +0,0 @@ -# Task Specification: TASK-007 - -## Implement Alert System - -**Phase:** 2 - Detection & Response -**Priority:** High -**Estimated Effort:** 2-3 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Implement a comprehensive alert system for security events. The alert system will generate alerts from rule matches, handle deduplication, and support multiple notification channels (Slack, email, webhook). - ---- - -## Requirements - -### 1. Alert Generation - -Create alert generation from: -- Rule match results -- Threat score thresholds -- Pattern detection -- Manual alert creation - -### 2. Alert Data Model - -Define alert structure with: -- Alert ID (UUID) -- Severity (Info, Low, Medium, High, Critical) -- Source event reference -- Rule/signature that triggered -- Timestamp -- Status (New, Acknowledged, Resolved) -- Metadata (container ID, process info, etc.) - -### 3. Alert Deduplication - -Implement deduplication with: -- Time-window based deduplication -- Fingerprinting (hash of alert properties) -- Aggregation of similar alerts -- Configurable dedup windows - -### 4. Notification Channels - -Implement notification providers: -- **Slack** - Webhook-based notifications -- **Email** - SMTP-based notifications -- **Webhook** - Generic HTTP webhook -- **Console** - Log-based notifications (for testing) - -### 5. Alert Management - -Provide alert management: -- Alert storage (in-memory + database ready) -- Alert querying and filtering -- Status updates (acknowledge, resolve) -- Alert statistics - ---- - -## TDD Tests to Create - -### Test File: `tests/alerting/alert_test.rs` - -```rust -#[test] -fn test_alert_creation() -#[test] -fn test_alert_id_generation() -#[test] -fn test_alert_severity_levels() -#[test] -fn test_alert_status_transitions() -#[test] -fn test_alert_fingerprint() -``` - -### Test File: `tests/alerting/alert_manager_test.rs` - -```rust -#[test] -fn test_alert_manager_creation() -#[test] -fn test_alert_generation_from_rule() -#[test] -fn test_alert_generation_from_threshold() -#[test] -fn test_alert_storage() -#[test] -fn test_alert_querying() -#[test] -fn test_alert_acknowledgment() -#[test] -fn test_alert_resolution() -``` - -### Test File: `tests/alerting/deduplication_test.rs` - -```rust -#[test] -fn test_deduplication_fingerprint() -#[test] -fn test_deduplication_time_window() -#[test] -fn test_deduplication_aggregation() -#[test] -fn test_deduplication_disabled() -``` - -### Test File: `tests/alerting/notifications_test.rs` - -```rust -#[test] -fn test_slack_notification() -#[test] -fn test_email_notification() -#[test] -fn test_webhook_notification() -#[test] -fn test_console_notification() -#[test] -fn test_notification_routing() -``` - ---- - -## Implementation Files - -### Alert System (`src/alerting/`) - -``` -src/alerting/ -├── mod.rs -├── alert.rs (NEW - alert data model) -├── manager.rs (NEW - alert management) -├── dedup.rs (from TASK-005, enhance) -├── notifications.rs (from TASK-005, enhance) -├── channels/ -│ ├── mod.rs -│ ├── slack.rs -│ ├── email.rs -│ ├── webhook.rs -│ └── console.rs -└── storage.rs (NEW - alert storage) -``` - ---- - -## Acceptance Criteria - -- [ ] Alert data model implemented -- [ ] Alert generation from rules working -- [ ] Deduplication with time windows -- [ ] 4 notification channels implemented -- [ ] Alert storage and querying -- [ ] Status management (new, ack, resolved) -- [ ] All tests passing (target: 30+ tests) -- [ ] Documentation complete - ---- - -*Created: 2026-03-13* diff --git a/docs/tasks/TASK-008-SUMMARY.md b/docs/tasks/TASK-008-SUMMARY.md deleted file mode 100644 index 982ad49..0000000 --- a/docs/tasks/TASK-008-SUMMARY.md +++ /dev/null @@ -1,449 +0,0 @@ -# TASK-008 Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** 2026-03-13 -**Developer:** Qwen Code - ---- - -## What Was Accomplished - -### 1. ✅ Firewall Backend Trait - -**File:** `src/firewall/backend.rs` - -#### FirewallBackend Trait -```rust -pub trait FirewallBackend: Send + Sync { - fn initialize(&mut self) -> Result<()>; - fn is_available(&self) -> bool; - fn block_ip(&self, ip: &str) -> Result<()>; - fn unblock_ip(&self, ip: &str) -> Result<()>; - fn block_port(&self, port: u16) -> Result<()>; - fn unblock_port(&self, port: u16) -> Result<()>; - fn block_container(&self, container_id: &str) -> Result<()>; - fn unblock_container(&self, container_id: &str) -> Result<()>; - fn name(&self) -> &str; -} -``` - -#### Supporting Types -- `FirewallRule` - Rule representation -- `FirewallTable` - Table representation -- `FirewallChain` - Chain representation - ---- - -### 2. ✅ nftables Backend - -**File:** `src/firewall/nftables.rs` - -#### NfTable Struct -```rust -pub struct NfTable { - pub family: String, - pub name: String, -} -``` - -#### NfChain Struct -```rust -pub struct NfChain { - pub table: NfTable, - pub name: String, - pub chain_type: String, -} -``` - -#### NfRule Struct -```rust -pub struct NfRule { - pub chain: NfChain, - pub rule_spec: String, -} -``` - -#### NfTablesBackend Methods -- `new() -> Result` - Create backend -- `create_table(table: &NfTable) -> Result<()>` -- `delete_table(table: &NfTable) -> Result<()>` -- `create_chain(chain: &NfChain) -> Result<()>` -- `delete_chain(chain: &NfChain) -> Result<()>` -- `add_rule(rule: &NfRule) -> Result<()>` -- `delete_rule(rule: &NfRule) -> Result<()>` -- `batch_add_rules(rules: &[NfRule]) -> Result<()>` -- `flush_chain(chain: &NfChain) -> Result<()>` -- `list_rules(chain: &NfChain) -> Result>` - -**Features:** -- Full nftables management via `nft` command -- Batch rule updates -- Table and chain lifecycle management - ---- - -### 3. ✅ iptables Backend (Fallback) - -**File:** `src/firewall/iptables.rs` - -#### IptChain Struct -```rust -pub struct IptChain { - pub table: String, - pub name: String, -} -``` - -#### IptRule Struct -```rust -pub struct IptRule { - pub chain: IptChain, - pub rule_spec: String, -} -``` - -#### IptablesBackend Methods -- `new() -> Result` - Create backend -- `create_chain(chain: &IptChain) -> Result<()>` -- `delete_chain(chain: &IptChain) -> Result<()>` -- `add_rule(rule: &IptRule) -> Result<()>` -- `delete_rule(rule: &IptRule) -> Result<()>` -- `flush_chain(chain: &IptChain) -> Result<()>` -- `list_rules(chain: &IptChain) -> Result>` - -**Features:** -- iptables management via `iptables` command -- Fallback when nftables unavailable -- Implements `FirewallBackend` trait - ---- - -### 4. ✅ Container Quarantine - -**File:** `src/firewall/quarantine.rs` - -#### QuarantineState Enum -```rust -pub enum QuarantineState { - Quarantined, - Released, - Failed, -} -``` - -#### QuarantineInfo Struct -```rust -pub struct QuarantineInfo { - pub container_id: String, - pub quarantined_at: DateTime, - pub released_at: Option>, - pub state: QuarantineState, - pub reason: Option, -} -``` - -#### QuarantineManager Struct -```rust -pub struct QuarantineManager { - nft: Option, - states: Arc>>, - table_name: String, -} -``` - -**Methods:** -- `new() -> Result` - Create manager -- `quarantine(container_id: &str) -> Result<()>` - Quarantine container -- `release(container_id: &str) -> Result<()>` - Release from quarantine -- `rollback(container_id: &str) -> Result<()>` - Rollback quarantine -- `get_state(container_id: &str) -> Option` - Get state -- `get_quarantined_containers() -> Vec` - List quarantined -- `get_quarantine_info(container_id: &str) -> Option` - Get info -- `get_stats() -> QuarantineStats` - Get statistics - -#### QuarantineStats Struct -```rust -pub struct QuarantineStats { - pub currently_quarantined: u64, - pub total_quarantined: u64, - pub released: u64, - pub failed: u64, -} -``` - -**Features:** -- Thread-safe state tracking (Arc) -- nftables integration for network isolation -- Quarantine lifecycle management -- Statistics tracking - ---- - -### 5. ✅ Automated Response - -**File:** `src/firewall/response.rs` - -#### ResponseType Enum -```rust -pub enum ResponseType { - BlockIP(String), - BlockPort(u16), - QuarantineContainer(String), - KillProcess(u32), - LogAction(String), - SendAlert(String), - Custom(String), -} -``` - -#### ResponseAction Struct -```rust -pub struct ResponseAction { - action_type: ResponseType, - description: String, - max_retries: u32, - retry_delay_ms: u64, -} -``` - -**Methods:** -- `new(action_type, description) -> Self` -- `from_alert(alert: &Alert, action_type) -> Self` -- `set_retry_config(max_retries, retry_delay_ms)` -- `execute() -> Result<()>` -- `execute_with_retry() -> Result<()>` - -#### ResponseChain Struct -```rust -pub struct ResponseChain { - name: String, - actions: Vec, - stop_on_failure: bool, -} -``` - -**Methods:** -- `new(name) -> Self` -- `add_action(action: ResponseAction)` -- `set_stop_on_failure(stop: bool)` -- `execute() -> Result<()>` - -#### ResponseExecutor Struct -```rust -pub struct ResponseExecutor { - log: Arc>>, -} -``` - -**Methods:** -- `new() -> Result` -- `execute(action: &ResponseAction) -> Result<()>` -- `execute_chain(chain: &ResponseChain) -> Result<()>` -- `get_log() -> Vec` -- `clear_log()` - -#### ResponseLog Struct -```rust -pub struct ResponseLog { - action_name: String, - success: bool, - error: Option, - timestamp: DateTime, -} -``` - -**Features:** -- Multiple response action types -- Retry logic with configurable delays -- Action chaining -- Execution logging -- Audit trail - ---- - -## Test Coverage - -### Tests Created: 25+ - -| Test File | Tests | Status | -|-----------|-------|--------| -| `nftables_test.rs` | 7 | ✅ Complete | -| `iptables_test.rs` | 6 | ✅ Complete | -| `quarantine_test.rs` | 8 | ✅ Complete | -| `response_test.rs` | 13 | ✅ Complete | -| **Module Tests** | 10+ | ✅ Complete | -| **Total** | **44+** | | - -### Test Coverage by Category - -| Category | Tests | -|----------|-------| -| nftables | 7 | -| iptables | 6 | -| Quarantine | 8 | -| Response | 13 | -| Module Tests | 10 | - ---- - -## Module Structure - -``` -src/firewall/ -├── mod.rs ✅ Updated exports -├── backend.rs ✅ Firewall trait -├── nftables.rs ✅ nftables backend -├── iptables.rs ✅ iptables fallback -├── quarantine.rs ✅ Container quarantine -└── response.rs ✅ Automated response -``` - ---- - -## Code Quality - -### Design Patterns -- **Strategy Pattern** - FirewallBackend trait for different backends -- **Command Pattern** - ResponseAction for encapsulating actions -- **Chain of Responsibility** - ResponseChain for action sequences -- **State Pattern** - QuarantineState for lifecycle - -### Thread Safety -- `Arc>` for shared state -- Safe concurrent access to quarantine states -- Thread-safe response logging - -### Error Handling -- `anyhow::Result` for fallible operations -- Graceful handling of missing tools (nft, iptables) -- Retry logic for transient failures - ---- - -## Integration Points - -### With Alert System -```rust -use stackdog::firewall::{ResponseAction, ResponseType}; -use stackdog::alerting::Alert; - -// Create response from alert -let action = ResponseAction::from_alert( - &alert, - ResponseType::QuarantineContainer(container_id.to_string()), -); - -let mut executor = ResponseExecutor::new()?; -executor.execute(&action)?; -``` - -### With Rule Engine -```rust -use stackdog::rules::RuleEngine; -use stackdog::firewall::{ResponseChain, ResponseAction, ResponseType}; - -// Create automated response chain -let mut chain = ResponseChain::new("threat_response"); -chain.add_action(ResponseAction::new( - ResponseType::LogAction("Threat detected".to_string()), - "Log threat".to_string(), -)); -chain.add_action(ResponseAction::new( - ResponseType::QuarantineContainer(container_id), - "Quarantine container".to_string(), -)); - -// Execute on rule match -if rule_matched { - executor.execute_chain(&chain)?; -} -``` - ---- - -## Usage Example - -```rust -use stackdog::firewall::{ - NfTablesBackend, NfTable, NfChain, NfRule, - QuarantineManager, ResponseAction, ResponseType, -}; - -// Setup nftables -let nft = NfTablesBackend::new()?; -let table = NfTable::new("inet", "stackdog"); -nft.create_table(&table)?; - -let chain = NfChain::new(&table, "input", "filter"); -nft.create_chain(&chain)?; - -// Add rule -let rule = NfRule::new(&chain, "tcp dport 22 drop"); -nft.add_rule(&rule)?; - -// Quarantine container -let mut quarantine = QuarantineManager::new()?; -quarantine.quarantine("abc123")?; - -// Automated response -let action = ResponseAction::new( - ResponseType::BlockIP("192.168.1.100".to_string()), - "Block malicious IP".to_string(), -); - -let mut executor = ResponseExecutor::new()?; -executor.execute(&action)?; - -// Get statistics -let stats = quarantine.get_stats(); -println!("Quarantined: {}", stats.currently_quarantined); -``` - ---- - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| nftables backend implemented | ✅ Complete | -| iptables fallback working | ✅ Complete | -| Container quarantine functional | ✅ Complete | -| Automated response actions | ✅ Complete | -| Response logging and audit | ✅ Complete | -| All tests passing (target: 25+ tests) | ✅ 44+ tests | -| Documentation complete | ✅ Complete | - ---- - -## Files Modified/Created - -### Created (5 files) -- `src/firewall/backend.rs` - Firewall trait -- `src/firewall/nftables.rs` - nftables backend -- `src/firewall/iptables.rs` - iptables fallback -- `src/firewall/quarantine.rs` - Container quarantine -- `src/firewall/response.rs` - Automated response -- `tests/firewall/nftables_test.rs` - nftables tests -- `tests/firewall/iptables_test.rs` - iptables tests -- `tests/firewall/quarantine_test.rs` - Quarantine tests -- `tests/firewall/response_test.rs` - Response tests - -### Modified -- `src/firewall/mod.rs` - Updated exports -- `src/lib.rs` - Added firewall re-exports -- `tests/firewall/mod.rs` - Added test modules - ---- - -## Total Project Stats After TASK-008 - -| Metric | Count | -|--------|-------| -| **Total Tests** | 373+ | -| **Files Created** | 85+ | -| **Lines of Code** | 11500+ | -| **Documentation** | 22 files | - ---- - -*Task completed: 2026-03-13* diff --git a/docs/tasks/TASK-008.md b/docs/tasks/TASK-008.md deleted file mode 100644 index 7e19b41..0000000 --- a/docs/tasks/TASK-008.md +++ /dev/null @@ -1,153 +0,0 @@ -# Task Specification: TASK-008 - -## Implement Firewall Integration - -**Phase:** 3 - Response & Automation -**Priority:** High -**Estimated Effort:** 3-4 days -**Status:** 🟢 In Progress - ---- - -## Objective - -Implement automated threat response through firewall management. This includes nftables backend, iptables fallback, container quarantine mechanisms, and automated response actions. - ---- - -## Requirements - -### 1. nftables Backend - -Implement nftables management: -- Table and chain creation -- Rule addition/removal -- Batch updates for performance -- Atomic rule changes -- Rule listing and inspection - -### 2. iptables Fallback - -Implement iptables support: -- Rule management -- Chain creation -- Fallback when nftables unavailable - -### 3. Container Quarantine - -Implement container isolation: -- Network isolation for containers -- Block all ingress/egress traffic -- Allow only management traffic -- Quarantine state tracking -- Rollback mechanism - -### 4. Automated Response - -Implement response automation: -- Trigger response from alerts -- Configurable response actions -- Response logging and audit -- Action retry logic - ---- - -## TDD Tests to Create - -### Test File: `tests/firewall/nftables_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_nft_table_creation() -#[test] -#[ignore = "requires root"] -fn test_nft_chain_creation() -#[test] -#[ignore = "requires root"] -fn test_nft_rule_addition() -#[test] -#[ignore = "requires root"] -fn test_nft_rule_removal() -#[test] -#[ignore = "requires root"] -fn test_nft_batch_update() -``` - -### Test File: `tests/firewall/iptables_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_ipt_rule_addition() -#[test] -#[ignore = "requires root"] -fn test_ipt_rule_removal() -#[test] -#[ignore = "requires root"] -fn test_ipt_chain_creation() -``` - -### Test File: `tests/firewall/quarantine_test.rs` - -```rust -#[test] -#[ignore = "requires root"] -fn test_container_quarantine() -#[test] -#[ignore = "requires root"] -fn test_container_release() -#[test] -#[ignore = "requires root"] -fn test_quarantine_state_tracking() -#[test] -#[ignore = "requires root"] -fn test_quarantine_rollback() -``` - -### Test File: `tests/firewall/response_test.rs` - -```rust -#[test] -fn test_response_action_creation() -#[test] -fn test_response_action_execution() -#[test] -fn test_response_chain() -#[test] -fn test_response_retry() -#[test] -fn test_response_logging() -``` - ---- - -## Implementation Files - -### Firewall (`src/firewall/`) - -``` -src/firewall/ -├── mod.rs -├── nftables.rs (enhance from TASK-003) -├── iptables.rs (enhance from TASK-003) -├── quarantine.rs (enhance from TASK-003) -├── backend.rs (NEW - trait abstraction) -└── response.rs (NEW - automated response) -``` - ---- - -## Acceptance Criteria - -- [ ] nftables backend implemented -- [ ] iptables fallback working -- [ ] Container quarantine functional -- [ ] Automated response actions -- [ ] Response logging and audit -- [ ] All tests passing (target: 25+ tests) -- [ ] Documentation complete - ---- - -*Created: 2026-03-13* diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..11e9942 --- /dev/null +++ b/install.sh @@ -0,0 +1,148 @@ +#!/bin/sh +# Stackdog Security — install script +# +# Usage: +# curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash +# curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash -s -- --version v0.2.0 +# +# Installs the stackdog binary to /usr/local/bin. +# Requires: curl, tar, sha256sum (or shasum), Linux x86_64 or aarch64. + +set -eu + +REPO="vsilent/stackdog" +INSTALL_DIR="/usr/local/bin" +BINARY_NAME="stackdog" + +# --- helpers ---------------------------------------------------------------- + +info() { printf '\033[1;32m▸ %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33m⚠ %s\033[0m\n' "$*"; } +error() { printf '\033[1;31m✖ %s\033[0m\n' "$*" >&2; exit 1; } + +need_cmd() { + if ! command -v "$1" > /dev/null 2>&1; then + error "Required command not found: $1" + fi +} + +# --- detect platform -------------------------------------------------------- + +detect_platform() { + OS="$(uname -s)" + ARCH="$(uname -m)" + + case "$OS" in + Linux) OS="linux" ;; + *) error "Unsupported OS: $OS. Stackdog binaries are available for Linux only." ;; + esac + + case "$ARCH" in + x86_64|amd64) ARCH="x86_64" ;; + aarch64|arm64) ARCH="aarch64" ;; + *) error "Unsupported architecture: $ARCH. Supported: x86_64, aarch64." ;; + esac + + PLATFORM="${OS}-${ARCH}" +} + +# --- resolve version -------------------------------------------------------- + +resolve_version() { + if [ -n "${VERSION:-}" ]; then + # strip leading v if present for consistency + VERSION="$(echo "$VERSION" | sed 's/^v//')" + TAG="v${VERSION}" + return + fi + + info "Fetching latest release..." + TAG="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" \ + | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/')" + + if [ -z "$TAG" ]; then + error "Could not determine latest release. Specify a version with --version" + fi + + VERSION="$(echo "$TAG" | sed 's/^v//')" +} + +# --- download & verify ------------------------------------------------------ + +download_and_install() { + TARBALL="${BINARY_NAME}-${PLATFORM}.tar.gz" + CHECKSUM_FILE="${TARBALL}.sha256" + DOWNLOAD_URL="https://github.com/${REPO}/releases/download/${TAG}/${TARBALL}" + CHECKSUM_URL="https://github.com/${REPO}/releases/download/${TAG}/${CHECKSUM_FILE}" + + TMPDIR="$(mktemp -d)" + trap 'rm -rf "$TMPDIR"' EXIT + + info "Downloading stackdog ${VERSION} for ${PLATFORM}..." + curl -fsSL -o "${TMPDIR}/${TARBALL}" "$DOWNLOAD_URL" \ + || error "Download failed. Check that release ${TAG} exists at https://github.com/${REPO}/releases" + + info "Downloading checksum..." + curl -fsSL -o "${TMPDIR}/${CHECKSUM_FILE}" "$CHECKSUM_URL" \ + || warn "Checksum file not available — skipping verification" + + # verify checksum if available + if [ -f "${TMPDIR}/${CHECKSUM_FILE}" ]; then + info "Verifying checksum..." + EXPECTED="$(awk '{print $1}' "${TMPDIR}/${CHECKSUM_FILE}")" + if command -v sha256sum > /dev/null 2>&1; then + ACTUAL="$(sha256sum "${TMPDIR}/${TARBALL}" | awk '{print $1}')" + elif command -v shasum > /dev/null 2>&1; then + ACTUAL="$(shasum -a 256 "${TMPDIR}/${TARBALL}" | awk '{print $1}')" + else + warn "sha256sum/shasum not found — skipping checksum verification" + ACTUAL="$EXPECTED" + fi + + if [ "$EXPECTED" != "$ACTUAL" ]; then + error "Checksum mismatch!\n expected: ${EXPECTED}\n actual: ${ACTUAL}" + fi + fi + + info "Extracting..." + tar -xzf "${TMPDIR}/${TARBALL}" -C "${TMPDIR}" + + info "Installing to ${INSTALL_DIR}/${BINARY_NAME}..." + install -m 755 "${TMPDIR}/${BINARY_NAME}" "${INSTALL_DIR}/${BINARY_NAME}" +} + +# --- main ------------------------------------------------------------------- + +main() { + # parse args + while [ $# -gt 0 ]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --help|-h) + echo "Usage: install.sh [--version VERSION]" + echo "" + echo "Install stackdog binary to ${INSTALL_DIR}." + echo "" + echo "Options:" + echo " --version VERSION Install a specific version (e.g. v0.2.0)" + echo " --help Show this help" + exit 0 + ;; + *) error "Unknown option: $1" ;; + esac + done + + need_cmd curl + need_cmd tar + + detect_platform + resolve_version + download_and_install + + info "stackdog ${VERSION} installed successfully!" + echo "" + echo " Run: stackdog --help" + echo "" +} + +main "$@" diff --git a/src/alerting/notifications.rs b/src/alerting/notifications.rs index aa5f25a..d35d7e0 100644 --- a/src/alerting/notifications.rs +++ b/src/alerting/notifications.rs @@ -111,14 +111,39 @@ impl NotificationChannel { Ok(NotificationResult::Success("sent to console".to_string())) } - /// Send to Slack + /// Send to Slack via incoming webhook fn send_slack(&self, alert: &Alert, config: &NotificationConfig) -> Result { - // In production, this would make HTTP request to Slack webhook - // For now, just log - if config.slack_webhook().is_some() { - log::info!("Would send to Slack: {}", alert.message()); - Ok(NotificationResult::Success("sent to Slack".to_string())) + if let Some(webhook_url) = config.slack_webhook() { + let payload = build_slack_message(alert); + log::debug!("Sending Slack notification to webhook"); + log::trace!("Slack payload: {}", payload); + + // Blocking HTTP POST — notification sending is synchronous in this codebase + let client = reqwest::blocking::Client::new(); + match client + .post(webhook_url) + .header("Content-Type", "application/json") + .body(payload) + .send() + { + Ok(resp) => { + if resp.status().is_success() { + log::info!("Slack notification sent successfully"); + Ok(NotificationResult::Success("sent to Slack".to_string())) + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + log::warn!("Slack API returned {}: {}", status, body); + Ok(NotificationResult::Failure(format!("Slack returned {}: {}", status, body))) + } + } + Err(e) => { + log::warn!("Failed to send Slack notification: {}", e); + Ok(NotificationResult::Failure(format!("Slack request failed: {}", e))) + } + } } else { + log::debug!("Slack webhook not configured, skipping"); Ok(NotificationResult::Failure("Slack webhook not configured".to_string())) } } @@ -211,27 +236,19 @@ pub fn severity_to_slack_color(severity: AlertSeverity) -> &'static str { /// Build Slack message payload pub fn build_slack_message(alert: &Alert) -> String { - format!( - r#"{{ - "text": "Security Alert", - "attachments": [{{ - "color": "{}", - "title": "{:?} ", - "text": "{}", - "fields": [ - {{"title": "Severity", "value": "{}", "short": true}}, - {{"title": "Status", "value": "{}", "short": true}}, - {{"title": "Time", "value": "{}", "short": true}} - ] - }}] - }}"#, - severity_to_slack_color(alert.severity()), - alert.alert_type(), - alert.message(), - alert.severity(), - alert.status(), - alert.timestamp() - ) + serde_json::json!({ + "text": "🐕 Stackdog Security Alert", + "attachments": [{ + "color": severity_to_slack_color(alert.severity()), + "title": format!("{:?}", alert.alert_type()), + "text": alert.message(), + "fields": [ + {"title": "Severity", "value": alert.severity().to_string(), "short": true}, + {"title": "Status", "value": alert.status().to_string(), "short": true}, + {"title": "Time", "value": alert.timestamp().to_rfc3339(), "short": true} + ] + }] + }).to_string() } /// Build webhook payload diff --git a/src/api/logs.rs b/src/api/logs.rs new file mode 100644 index 0000000..9963c33 --- /dev/null +++ b/src/api/logs.rs @@ -0,0 +1,277 @@ +//! Log sources and summaries API endpoints + +use actix_web::{web, HttpResponse, Responder}; +use serde::Deserialize; +use crate::database::connection::DbPool; +use crate::database::repositories::log_sources; +use crate::sniff::discovery::{LogSource, LogSourceType}; + +/// Query parameters for summary filtering +#[derive(Debug, Deserialize)] +pub struct SummaryQuery { + source_id: Option, +} + +/// Request body for adding a custom log source +#[derive(Debug, Deserialize)] +pub struct AddSourceRequest { + pub path: String, + pub name: Option, +} + +/// List all discovered log sources +/// +/// GET /api/logs/sources +pub async fn list_sources(pool: web::Data) -> impl Responder { + match log_sources::list_log_sources(&pool) { + Ok(sources) => HttpResponse::Ok().json(sources), + Err(e) => { + log::error!("Failed to list log sources: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to list log sources" + })) + } + } +} + +/// Get a single log source by path +/// +/// GET /api/logs/sources/{path} +pub async fn get_source(pool: web::Data, path: web::Path) -> impl Responder { + match log_sources::get_log_source_by_path(&pool, &path) { + Ok(Some(source)) => HttpResponse::Ok().json(source), + Ok(None) => HttpResponse::NotFound().json(serde_json::json!({ + "error": "Log source not found" + })), + Err(e) => { + log::error!("Failed to get log source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to get log source" + })) + } + } +} + +/// Manually add a custom log source +/// +/// POST /api/logs/sources +pub async fn add_source( + pool: web::Data, + body: web::Json, +) -> impl Responder { + let name = body.name.clone().unwrap_or_else(|| body.path.clone()); + let source = LogSource::new(LogSourceType::CustomFile, body.path.clone(), name); + + match log_sources::upsert_log_source(&pool, &source) { + Ok(_) => HttpResponse::Created().json(source), + Err(e) => { + log::error!("Failed to add log source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to add log source" + })) + } + } +} + +/// Delete a log source +/// +/// DELETE /api/logs/sources/{path} +pub async fn delete_source(pool: web::Data, path: web::Path) -> impl Responder { + match log_sources::delete_log_source(&pool, &path) { + Ok(_) => HttpResponse::NoContent().finish(), + Err(e) => { + log::error!("Failed to delete log source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to delete log source" + })) + } + } +} + +/// List AI-generated log summaries +/// +/// GET /api/logs/summaries +pub async fn list_summaries( + pool: web::Data, + query: web::Query, +) -> impl Responder { + let source_id = query.source_id.as_deref().unwrap_or(""); + if source_id.is_empty() { + // List all summaries — check each known source + match log_sources::list_log_sources(&pool) { + Ok(sources) => { + let mut all_summaries = Vec::new(); + for source in &sources { + if let Ok(summaries) = log_sources::list_summaries_for_source(&pool, &source.path_or_id) { + all_summaries.extend(summaries); + } + } + HttpResponse::Ok().json(all_summaries) + } + Err(e) => { + log::error!("Failed to list summaries: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to list summaries" + })) + } + } + } else { + match log_sources::list_summaries_for_source(&pool, source_id) { + Ok(summaries) => HttpResponse::Ok().json(summaries), + Err(e) => { + log::error!("Failed to list summaries for source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to list summaries" + })) + } + } + } +} + +/// Configure log API routes +pub fn configure_routes(cfg: &mut web::ServiceConfig) { + cfg.service( + web::scope("/api/logs") + .route("/sources", web::get().to(list_sources)) + .route("/sources", web::post().to(add_source)) + .route("/sources/{path}", web::get().to(get_source)) + .route("/sources/{path}", web::delete().to(delete_source)) + .route("/summaries", web::get().to(list_summaries)) + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use actix_web::{test, App}; + use crate::database::connection::{create_pool, init_database}; + + fn setup_pool() -> DbPool { + let pool = create_pool(":memory:").unwrap(); + init_database(&pool).unwrap(); + pool + } + + #[actix_rt::test] + async fn test_list_sources_empty() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get().uri("/api/logs/sources").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + } + + #[actix_rt::test] + async fn test_add_source() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let body = serde_json::json!({ "path": "/var/log/test.log", "name": "Test Log" }); + let req = test::TestRequest::post() + .uri("/api/logs/sources") + .set_json(&body) + .to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 201); + } + + #[actix_rt::test] + async fn test_add_and_list_sources() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + // Add a source + let body = serde_json::json!({ "path": "/var/log/app.log" }); + let req = test::TestRequest::post() + .uri("/api/logs/sources") + .set_json(&body) + .to_request(); + test::call_service(&app, req).await; + + // List sources + let req = test::TestRequest::get().uri("/api/logs/sources").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + + let body: Vec = test::read_body_json(resp).await; + assert_eq!(body.len(), 1); + } + + #[actix_rt::test] + async fn test_get_source_not_found() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get().uri("/api/logs/sources/nonexistent").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 404); + } + + #[actix_rt::test] + async fn test_delete_source() { + let pool = setup_pool(); + + // Add source directly via repository (avoids route path issues) + let source = LogSource::new(LogSourceType::CustomFile, "test-delete.log".into(), "Test Delete".into()); + log_sources::upsert_log_source(&pool, &source).unwrap(); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::delete() + .uri("/api/logs/sources/test-delete.log") + .to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 204); + } + + #[actix_rt::test] + async fn test_list_summaries_empty() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get().uri("/api/logs/summaries").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + } + + #[actix_rt::test] + async fn test_list_summaries_filtered() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get() + .uri("/api/logs/summaries?source_id=test-source") + .to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + } +} diff --git a/src/api/mod.rs b/src/api/mod.rs index 754a6d5..6120aab 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,6 +7,7 @@ pub mod alerts; pub mod containers; pub mod threats; pub mod websocket; +pub mod logs; /// Marker struct for module tests pub struct ApiMarker; @@ -17,6 +18,7 @@ pub use alerts::configure_routes as configure_alerts_routes; pub use containers::configure_routes as configure_containers_routes; pub use threats::configure_routes as configure_threats_routes; pub use websocket::configure_routes as configure_websocket_routes; +pub use logs::configure_routes as configure_logs_routes; /// Configure all API routes pub fn configure_all_routes(cfg: &mut actix_web::web::ServiceConfig) { @@ -25,4 +27,5 @@ pub fn configure_all_routes(cfg: &mut actix_web::web::ServiceConfig) { configure_containers_routes(cfg); configure_threats_routes(cfg); configure_websocket_routes(cfg); + configure_logs_routes(cfg); } diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..ea26fcc --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,173 @@ +//! CLI argument parsing for Stackdog +//! +//! Defines the command-line interface using clap derive macros. +//! Supports `serve` (HTTP server) and `sniff` (log analysis) subcommands. + +use clap::{Parser, Subcommand}; + +/// Stackdog Security — Docker & Linux server security platform +#[derive(Parser, Debug)] +#[command(name = "stackdog", version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Option, +} + +/// Available subcommands +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Start the HTTP API server (default behavior) + Serve, + + /// Sniff and analyze logs from Docker containers and system sources + Sniff { + /// Run a single scan/analysis pass, then exit + #[arg(long)] + once: bool, + + /// Consume logs: archive to zstd, then purge originals to free disk + #[arg(long)] + consume: bool, + + /// Output directory for consumed logs + #[arg(long, default_value = "./stackdog-logs/")] + output: String, + + /// Additional log file paths to watch (comma-separated) + #[arg(long)] + sources: Option, + + /// Poll interval in seconds + #[arg(long, default_value = "30")] + interval: u64, + + /// AI provider: "openai", "ollama", or "candle" + #[arg(long)] + ai_provider: Option, + + /// AI model name (e.g. "gpt-4o-mini", "qwen2.5-coder:latest", "llama3") + #[arg(long)] + ai_model: Option, + + /// AI API URL (e.g. "http://localhost:11434/v1" for Ollama) + #[arg(long)] + ai_api_url: Option, + + /// Slack webhook URL for alert notifications + #[arg(long)] + slack_webhook: Option, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[test] + fn test_no_subcommand_defaults_to_none() { + let cli = Cli::parse_from(["stackdog"]); + assert!(cli.command.is_none(), "No subcommand should yield None (default to serve)"); + } + + #[test] + fn test_serve_subcommand() { + let cli = Cli::parse_from(["stackdog", "serve"]); + assert!(matches!(cli.command, Some(Command::Serve))); + } + + #[test] + fn test_sniff_subcommand_defaults() { + let cli = Cli::parse_from(["stackdog", "sniff"]); + match cli.command { + Some(Command::Sniff { once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook }) => { + assert!(!once); + assert!(!consume); + assert_eq!(output, "./stackdog-logs/"); + assert!(sources.is_none()); + assert_eq!(interval, 30); + assert!(ai_provider.is_none()); + assert!(ai_model.is_none()); + assert!(ai_api_url.is_none()); + assert!(slack_webhook.is_none()); + } + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_once_flag() { + let cli = Cli::parse_from(["stackdog", "sniff", "--once"]); + match cli.command { + Some(Command::Sniff { once, .. }) => assert!(once), + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_consume_flag() { + let cli = Cli::parse_from(["stackdog", "sniff", "--consume"]); + match cli.command { + Some(Command::Sniff { consume, .. }) => assert!(consume), + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_all_options() { + let cli = Cli::parse_from([ + "stackdog", "sniff", + "--once", + "--consume", + "--output", "/tmp/logs/", + "--sources", "/var/log/syslog,/var/log/auth.log", + "--interval", "60", + "--ai-provider", "openai", + "--ai-model", "gpt-4o-mini", + "--ai-api-url", "https://api.openai.com/v1", + "--slack-webhook", "https://hooks.slack.com/services/T/B/xxx", + ]); + match cli.command { + Some(Command::Sniff { once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook }) => { + assert!(once); + assert!(consume); + assert_eq!(output, "/tmp/logs/"); + assert_eq!(sources.unwrap(), "/var/log/syslog,/var/log/auth.log"); + assert_eq!(interval, 60); + assert_eq!(ai_provider.unwrap(), "openai"); + assert_eq!(ai_model.unwrap(), "gpt-4o-mini"); + assert_eq!(ai_api_url.unwrap(), "https://api.openai.com/v1"); + assert_eq!(slack_webhook.unwrap(), "https://hooks.slack.com/services/T/B/xxx"); + } + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_candle_provider() { + let cli = Cli::parse_from(["stackdog", "sniff", "--ai-provider", "candle"]); + match cli.command { + Some(Command::Sniff { ai_provider, .. }) => { + assert_eq!(ai_provider.unwrap(), "candle"); + } + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_ollama_provider_and_model() { + let cli = Cli::parse_from([ + "stackdog", "sniff", + "--once", + "--ai-provider", "ollama", + "--ai-model", "qwen2.5-coder:latest", + ]); + match cli.command { + Some(Command::Sniff { ai_provider, ai_model, .. }) => { + assert_eq!(ai_provider.unwrap(), "ollama"); + assert_eq!(ai_model.unwrap(), "qwen2.5-coder:latest"); + } + _ => panic!("Expected Sniff command"), + } + } +} diff --git a/src/collectors/ebpf/container.rs b/src/collectors/ebpf/container.rs index 9cd568c..98de118 100644 --- a/src/collectors/ebpf/container.rs +++ b/src/collectors/ebpf/container.rs @@ -196,16 +196,16 @@ mod tests { #[test] fn test_parse_docker_cgroup() { - let cgroup = "12:memory:/docker/abc123def456789012345678901234567890"; + let cgroup = "12:memory:/docker/abc123def456abc123def456abc123def456abc123def456abc123def456abcd"; let result = ContainerDetector::parse_container_from_cgroup(cgroup); - assert_eq!(result, Some("abc123def456789012345678901234567890".to_string())); + assert_eq!(result, Some("abc123def456abc123def456abc123def456abc123def456abc123def456abcd".to_string())); } - + #[test] fn test_parse_kubernetes_cgroup() { - let cgroup = "11:cpu:/kubepods/pod123/def456abc789012345678901234567890"; + let cgroup = "11:cpu:/kubepods/pod123/def456abc123def456abc123def456abc123def456abc123def456abc123def4"; let result = ContainerDetector::parse_container_from_cgroup(cgroup); - assert_eq!(result, Some("def456abc789012345678901234567890".to_string())); + assert_eq!(result, Some("def456abc123def456abc123def456abc123def456abc123def456abc123def4".to_string())); } #[test] @@ -215,6 +215,7 @@ mod tests { assert_eq!(result, None); } + #[cfg(target_os = "linux")] #[test] fn test_validate_valid_container_id() { let detector = ContainerDetector::new().unwrap(); @@ -226,6 +227,7 @@ mod tests { assert!(detector.validate_container_id("abc123def456")); } + #[cfg(target_os = "linux")] #[test] fn test_validate_invalid_container_id() { let detector = ContainerDetector::new().unwrap(); diff --git a/src/collectors/ebpf/enrichment.rs b/src/collectors/ebpf/enrichment.rs index 00df2a6..fcbde6c 100644 --- a/src/collectors/ebpf/enrichment.rs +++ b/src/collectors/ebpf/enrichment.rs @@ -133,7 +133,7 @@ pub fn normalize_timestamp(ts: chrono::DateTime) -> chrono::DateTim #[cfg(test)] mod tests { use super::*; - + use chrono::Utc; #[test] fn test_enricher_creation() { let enricher = EventEnricher::new(); diff --git a/src/collectors/ebpf/loader.rs b/src/collectors/ebpf/loader.rs index 516b7d5..5838f1d 100644 --- a/src/collectors/ebpf/loader.rs +++ b/src/collectors/ebpf/loader.rs @@ -97,12 +97,15 @@ impl EbpfLoader { if _bytes.is_empty() { return Err(LoadError::LoadFailed("Empty program bytes".to_string())); } - - // TODO: Implement actual loading when eBPF programs are ready - // For now, this is a stub that will be implemented in TASK-004 + + let bpf = aya::Bpf::load(_bytes) + .map_err(|e| LoadError::LoadFailed(e.to_string()))?; + self.bpf = Some(bpf); + + log::info!("eBPF program loaded ({} bytes)", _bytes.len()); Ok(()) } - + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { Err(LoadError::NotLinux) @@ -132,23 +135,80 @@ impl EbpfLoader { pub fn attach_program(&mut self, _program_name: &str) -> Result<(), LoadError> { #[cfg(all(target_os = "linux", feature = "ebpf"))] { - // TODO: Implement actual attachment - // For now, just mark as attached + let (category, tp_name) = program_to_tracepoint(_program_name) + .ok_or_else(|| LoadError::ProgramNotFound( + format!("No tracepoint mapping for '{}'", _program_name) + ))?; + + let bpf = self.bpf.as_mut() + .ok_or_else(|| LoadError::LoadFailed( + "No eBPF program loaded; call load_program_from_bytes first".to_string() + ))?; + + let prog: &mut aya::programs::TracePoint = bpf + .program_mut(_program_name) + .ok_or_else(|| LoadError::ProgramNotFound(_program_name.to_string()))? + .try_into() + .map_err(|e: aya::programs::ProgramError| LoadError::AttachFailed(e.to_string()))?; + + prog.load() + .map_err(|e| LoadError::AttachFailed(format!("load '{}': {}", _program_name, e)))?; + + prog.attach(category, tp_name) + .map_err(|e| LoadError::AttachFailed( + format!("attach '{}/{}': {}", category, tp_name, e) + ))?; + self.loaded_programs.insert( _program_name.to_string(), - ProgramInfo { - name: _program_name.to_string(), - attached: true, - }, + ProgramInfo { name: _program_name.to_string(), attached: true }, ); + + log::info!("eBPF program '{}' attached to {}/{}", _program_name, category, tp_name); Ok(()) } - + + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] + { + Err(LoadError::NotLinux) + } + } + + /// Attach all known syscall tracepoint programs + pub fn attach_all_programs(&mut self) -> Result<(), LoadError> { + #[cfg(all(target_os = "linux", feature = "ebpf"))] + { + for name in &["trace_execve", "trace_connect", "trace_openat", "trace_ptrace"] { + if let Err(e) = self.attach_program(name) { + log::warn!("Failed to attach '{}': {}", name, e); + } + } + Ok(()) + } + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { Err(LoadError::NotLinux) } } + + /// Extract the EVENTS ring buffer map from the loaded eBPF program. + /// Must be called after load_program_from_bytes and before the Bpf object is dropped. + #[cfg(all(target_os = "linux", feature = "ebpf"))] + pub fn take_ring_buf(&mut self) -> Result, LoadError> { + let bpf = self.bpf.as_mut() + .ok_or_else(|| LoadError::LoadFailed( + "No eBPF program loaded".to_string() + ))?; + + let map = bpf.take_map("EVENTS") + .ok_or_else(|| LoadError::LoadFailed( + "EVENTS ring buffer map not found in eBPF program".to_string() + ))?; + + aya::maps::RingBuf::try_from(map) + .map_err(|e| LoadError::LoadFailed(format!("Failed to create ring buffer: {}", e))) + } /// Detach a program pub fn detach_program(&mut self, program_name: &str) -> Result<(), LoadError> { @@ -201,8 +261,24 @@ impl EbpfLoader { } impl Default for EbpfLoader { - fn default() -> Result { - Self::new() + fn default() -> Self { + Self { + #[cfg(all(target_os = "linux", feature = "ebpf"))] + bpf: None, + loaded_programs: HashMap::new(), + kernel_version: None, + } + } +} + +/// Map program name to its tracepoint (category, name) for aya attachment. +fn program_to_tracepoint(name: &str) -> Option<(&'static str, &'static str)> { + match name { + "trace_execve" => Some(("syscalls", "sys_enter_execve")), + "trace_connect" => Some(("syscalls", "sys_enter_connect")), + "trace_openat" => Some(("syscalls", "sys_enter_openat")), + "trace_ptrace" => Some(("syscalls", "sys_enter_ptrace")), + _ => None, } } diff --git a/src/collectors/ebpf/programs/mod.rs b/src/collectors/ebpf/programs/mod.rs deleted file mode 100644 index 5988d70..0000000 --- a/src/collectors/ebpf/programs/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! eBPF programs module -//! -//! Contains eBPF program definitions - -// eBPF programs will be implemented in TASK-003 -// This module will contain: -// - Syscall tracepoint programs -// - Network monitoring programs -// - Container-specific programs diff --git a/src/collectors/ebpf/ring_buffer.rs b/src/collectors/ebpf/ring_buffer.rs index 1983a68..9c25b01 100644 --- a/src/collectors/ebpf/ring_buffer.rs +++ b/src/collectors/ebpf/ring_buffer.rs @@ -59,6 +59,11 @@ impl EventRingBuffer { self.capacity } + /// View events without consuming them + pub fn events(&self) -> &[SyscallEvent] { + &self.buffer + } + /// Clear the buffer pub fn clear(&mut self) { self.buffer.clear(); diff --git a/src/collectors/ebpf/syscall_monitor.rs b/src/collectors/ebpf/syscall_monitor.rs index 5f4828f..df92490 100644 --- a/src/collectors/ebpf/syscall_monitor.rs +++ b/src/collectors/ebpf/syscall_monitor.rs @@ -12,7 +12,10 @@ use crate::collectors::ebpf::container::ContainerDetector; pub struct SyscallMonitor { #[cfg(all(target_os = "linux", feature = "ebpf"))] loader: Option, - + + #[cfg(all(target_os = "linux", feature = "ebpf"))] + ring_buf: Option>, + running: bool, event_buffer: EventRingBuffer, enricher: EventEnricher, @@ -34,6 +37,7 @@ impl SyscallMonitor { Ok(Self { loader: Some(loader), + ring_buf: None, running: false, event_buffer: EventRingBuffer::with_capacity(8192), enricher, @@ -54,15 +58,36 @@ impl SyscallMonitor { if self.running { anyhow::bail!("Monitor is already running"); } - - // TODO: Actually start eBPF programs in TASK-004 - // For now, just mark as running + + if let Some(loader) = &mut self.loader { + let ebpf_path = "target/bpfel-unknown-none/release/stackdog"; + match loader.load_program_from_file(ebpf_path) { + Ok(()) => { + loader.attach_all_programs().unwrap_or_else(|e| { + log::warn!("Some eBPF programs failed to attach: {}", e); + }); + match loader.take_ring_buf() { + Ok(rb) => { self.ring_buf = Some(rb); } + Err(e) => { log::warn!("Failed to get eBPF ring buffer: {}", e); } + } + } + Err(e) => { + log::warn!( + "eBPF program not found at '{}': {}. \ + Running without kernel event collection — \ + build the eBPF crate first with `cargo build --release` \ + in the ebpf/ directory.", + ebpf_path, e + ); + } + } + } + self.running = true; - log::info!("Syscall monitor started"); Ok(()) } - + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { anyhow::bail!("SyscallMonitor is only available on Linux"); @@ -73,7 +98,10 @@ impl SyscallMonitor { pub fn stop(&mut self) -> Result<()> { self.running = false; self.event_buffer.clear(); - + #[cfg(all(target_os = "linux", feature = "ebpf"))] + { + self.ring_buf = None; + } log::info!("Syscall monitor stopped"); Ok(()) } @@ -90,25 +118,39 @@ impl SyscallMonitor { if !self.running { return Vec::new(); } - - // TODO: Actually poll eBPF ring buffer in TASK-004 - // For now, drain from internal buffer + + // Drain the eBPF ring buffer into the staging buffer + if let Some(rb) = &mut self.ring_buf { + while let Some(item) = rb.next() { + let bytes: &[u8] = &item; + if bytes.len() >= std::mem::size_of::() { + // SAFETY: We verified the byte length matches the struct size, + // and EbpfSyscallEvent is #[repr(C)] with no padding surprises. + let raw: super::types::EbpfSyscallEvent = unsafe { + std::ptr::read_unaligned( + bytes.as_ptr() as *const super::types::EbpfSyscallEvent + ) + }; + self.event_buffer.push(raw.to_syscall_event()); + } + } + } + + // Drain the staging buffer and enrich with /proc info let mut events = self.event_buffer.drain(); - - // Enrich events for event in &mut events { let _ = self.enricher.enrich(event); } - + events } - + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { Vec::new() } } - + /// Get events without consuming them pub fn peek_events(&self) -> &[SyscallEvent] { self.event_buffer.events() diff --git a/src/collectors/ebpf/types.rs b/src/collectors/ebpf/types.rs index f8ef26a..6e97d28 100644 --- a/src/collectors/ebpf/types.rs +++ b/src/collectors/ebpf/types.rs @@ -27,7 +27,7 @@ pub struct EbpfSyscallEvent { /// Event data union #[repr(C)] -#[derive(Debug, Clone, Copy)] +#[derive(Clone, Copy)] pub union EbpfEventData { /// execve data pub execve: ExecveData, @@ -41,6 +41,14 @@ pub union EbpfEventData { pub raw: [u8; 128], } +impl std::fmt::Debug for EbpfEventData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // SAFETY: raw is always a valid field in any union variant + let raw = unsafe { self.raw }; + write!(f, "EbpfEventData {{ raw: {:?} }}", &raw[..]) + } +} + impl Default for EbpfEventData { fn default() -> Self { Self { @@ -51,7 +59,7 @@ impl Default for EbpfEventData { /// execve-specific data #[repr(C)] -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] pub struct ExecveData { /// Filename length pub filename_len: u32, @@ -61,6 +69,12 @@ pub struct ExecveData { pub argc: u32, } +impl Default for ExecveData { + fn default() -> Self { + Self { filename_len: 0, filename: [0u8; 128], argc: 0 } + } +} + /// connect-specific data #[repr(C)] #[derive(Debug, Clone, Copy, Default)] @@ -75,7 +89,7 @@ pub struct ConnectData { /// openat-specific data #[repr(C)] -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] pub struct OpenatData { /// File path length pub path_len: u32, @@ -85,6 +99,12 @@ pub struct OpenatData { pub flags: u32, } +impl Default for OpenatData { + fn default() -> Self { + Self { path_len: 0, path: [0u8; 256], flags: 0 } + } +} + /// ptrace-specific data #[repr(C)] #[derive(Debug, Clone, Copy, Default)] diff --git a/src/database/connection.rs b/src/database/connection.rs index d64ab39..d98d619 100644 --- a/src/database/connection.rs +++ b/src/database/connection.rs @@ -108,6 +108,38 @@ pub fn init_database(pool: &DbPool) -> Result<()> { let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_containers_status ON containers_cache(status)", []); let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_containers_name ON containers_cache(name)", []); + + // Create log_sources table + conn.execute( + "CREATE TABLE IF NOT EXISTS log_sources ( + id TEXT PRIMARY KEY, + source_type TEXT NOT NULL, + path_or_id TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + discovered_at TEXT NOT NULL, + last_read_position INTEGER DEFAULT 0 + )", + [], + )?; + + // Create log_summaries table + conn.execute( + "CREATE TABLE IF NOT EXISTS log_summaries ( + id TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + summary_text TEXT NOT NULL, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + total_entries INTEGER DEFAULT 0, + error_count INTEGER DEFAULT 0, + warning_count INTEGER DEFAULT 0, + created_at TEXT NOT NULL + )", + [], + )?; + + let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_log_sources_type ON log_sources(source_type)", []); + let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_log_summaries_source ON log_summaries(source_id)", []); Ok(()) } diff --git a/src/database/mod.rs b/src/database/mod.rs index e9bbe45..c8fa512 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -7,3 +7,6 @@ pub mod repositories; pub use connection::{create_pool, init_database, DbPool}; pub use models::*; pub use repositories::alerts::*; + +/// Marker struct for module tests +pub struct DatabaseMarker; diff --git a/src/database/repositories/log_sources.rs b/src/database/repositories/log_sources.rs new file mode 100644 index 0000000..70e45fe --- /dev/null +++ b/src/database/repositories/log_sources.rs @@ -0,0 +1,308 @@ +//! Log sources repository using rusqlite +//! +//! Persists discovered log sources and AI summaries, following +//! the same pattern as the alerts repository. + +use rusqlite::params; +use anyhow::Result; +use crate::database::connection::DbPool; +use crate::sniff::discovery::{LogSource, LogSourceType}; +use chrono::Utc; + +/// Create or update a log source (upsert by path_or_id) +pub fn upsert_log_source(pool: &DbPool, source: &LogSource) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "INSERT INTO log_sources (id, source_type, path_or_id, name, discovered_at, last_read_position) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(path_or_id) DO UPDATE SET + name = excluded.name, + source_type = excluded.source_type", + params![ + source.id, + source.source_type.to_string(), + source.path_or_id, + source.name, + source.discovered_at.to_rfc3339(), + source.last_read_position as i64, + ], + )?; + Ok(()) +} + +/// List all registered log sources +pub fn list_log_sources(pool: &DbPool) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, source_type, path_or_id, name, discovered_at, last_read_position + FROM log_sources ORDER BY discovered_at DESC" + )?; + + let sources = stmt.query_map([], |row| { + let source_type_str: String = row.get(1)?; + let discovered_str: String = row.get(4)?; + let pos: i64 = row.get(5)?; + Ok(LogSource { + id: row.get(0)?, + source_type: LogSourceType::from_str(&source_type_str), + path_or_id: row.get(2)?, + name: row.get(3)?, + discovered_at: chrono::DateTime::parse_from_rfc3339(&discovered_str) + .map(|dt| dt.with_timezone(&Utc)) + .unwrap_or_else(|_| Utc::now()), + last_read_position: pos as u64, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + + Ok(sources) +} + +/// Get a log source by its path or container ID +pub fn get_log_source_by_path(pool: &DbPool, path_or_id: &str) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, source_type, path_or_id, name, discovered_at, last_read_position + FROM log_sources WHERE path_or_id = ?" + )?; + + let result = stmt.query_row(params![path_or_id], |row| { + let source_type_str: String = row.get(1)?; + let discovered_str: String = row.get(4)?; + let pos: i64 = row.get(5)?; + Ok(LogSource { + id: row.get(0)?, + source_type: LogSourceType::from_str(&source_type_str), + path_or_id: row.get(2)?, + name: row.get(3)?, + discovered_at: chrono::DateTime::parse_from_rfc3339(&discovered_str) + .map(|dt| dt.with_timezone(&Utc)) + .unwrap_or_else(|_| Utc::now()), + last_read_position: pos as u64, + }) + }); + + match result { + Ok(source) => Ok(Some(source)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(anyhow::anyhow!("Database error: {}", e)), + } +} + +/// Update the read position for a log source +pub fn update_read_position(pool: &DbPool, path_or_id: &str, position: u64) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "UPDATE log_sources SET last_read_position = ?1 WHERE path_or_id = ?2", + params![position as i64, path_or_id], + )?; + Ok(()) +} + +/// Delete a log source +pub fn delete_log_source(pool: &DbPool, path_or_id: &str) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "DELETE FROM log_sources WHERE path_or_id = ?", + params![path_or_id], + )?; + Ok(()) +} + +/// Store a log summary +pub fn create_log_summary( + pool: &DbPool, + source_id: &str, + summary_text: &str, + period_start: &str, + period_end: &str, + total_entries: i64, + error_count: i64, + warning_count: i64, +) -> Result { + let conn = pool.get()?; + let id = uuid::Uuid::new_v4().to_string(); + let now = Utc::now().to_rfc3339(); + + conn.execute( + "INSERT INTO log_summaries (id, source_id, summary_text, period_start, period_end, + total_entries, error_count, warning_count, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![id, source_id, summary_text, period_start, period_end, + total_entries, error_count, warning_count, now], + )?; + + Ok(id) +} + +/// List summaries for a source +pub fn list_summaries_for_source(pool: &DbPool, source_id: &str) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, source_id, summary_text, period_start, period_end, + total_entries, error_count, warning_count, created_at + FROM log_summaries WHERE source_id = ? ORDER BY created_at DESC" + )?; + + let rows = stmt.query_map(params![source_id], |row| { + Ok(LogSummaryRow { + id: row.get(0)?, + source_id: row.get(1)?, + summary_text: row.get(2)?, + period_start: row.get(3)?, + period_end: row.get(4)?, + total_entries: row.get(5)?, + error_count: row.get(6)?, + warning_count: row.get(7)?, + created_at: row.get(8)?, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + + Ok(rows) +} + +/// Database row for a log summary +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LogSummaryRow { + pub id: String, + pub source_id: String, + pub summary_text: String, + pub period_start: String, + pub period_end: String, + pub total_entries: i64, + pub error_count: i64, + pub warning_count: i64, + pub created_at: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::database::connection::{create_pool, init_database}; + + fn setup_test_db() -> DbPool { + let pool = create_pool(":memory:").unwrap(); + init_database(&pool).unwrap(); + pool + } + + #[test] + fn test_upsert_and_list_log_sources() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/test.log".into(), + "test.log".into(), + ); + + upsert_log_source(&pool, &source).unwrap(); + let sources = list_log_sources(&pool).unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].path_or_id, "/var/log/test.log"); + assert_eq!(sources[0].name, "test.log"); + } + + #[test] + fn test_upsert_deduplicates_by_path() { + let pool = setup_test_db(); + let source1 = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog-v1".into(), + ); + let source2 = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog-v2".into(), + ); + + upsert_log_source(&pool, &source1).unwrap(); + upsert_log_source(&pool, &source2).unwrap(); + + let sources = list_log_sources(&pool).unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].name, "syslog-v2"); + } + + #[test] + fn test_get_log_source_by_path() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::DockerContainer, + "container-abc123".into(), + "docker:myapp".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + + let found = get_log_source_by_path(&pool, "container-abc123").unwrap(); + assert!(found.is_some()); + assert_eq!(found.unwrap().name, "docker:myapp"); + + let not_found = get_log_source_by_path(&pool, "nonexistent").unwrap(); + assert!(not_found.is_none()); + } + + #[test] + fn test_update_read_position() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::CustomFile, + "/tmp/app.log".into(), + "app.log".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + + update_read_position(&pool, "/tmp/app.log", 4096).unwrap(); + + let updated = get_log_source_by_path(&pool, "/tmp/app.log").unwrap().unwrap(); + assert_eq!(updated.last_read_position, 4096); + } + + #[test] + fn test_delete_log_source() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/test.log".into(), + "test.log".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + assert_eq!(list_log_sources(&pool).unwrap().len(), 1); + + delete_log_source(&pool, "/var/log/test.log").unwrap(); + assert_eq!(list_log_sources(&pool).unwrap().len(), 0); + } + + #[test] + fn test_create_and_list_summaries() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + + let summary_id = create_log_summary( + &pool, + &source.id, + "System running normally. 3 warnings about disk space.", + "2026-03-30T12:00:00Z", + "2026-03-30T13:00:00Z", + 500, + 0, + 3, + ).unwrap(); + + assert!(!summary_id.is_empty()); + + let summaries = list_summaries_for_source(&pool, &source.id).unwrap(); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].total_entries, 500); + assert_eq!(summaries[0].warning_count, 3); + assert!(summaries[0].summary_text.contains("disk space")); + } +} diff --git a/src/database/repositories/mod.rs b/src/database/repositories/mod.rs index 92b469d..8f790f5 100644 --- a/src/database/repositories/mod.rs +++ b/src/database/repositories/mod.rs @@ -1,6 +1,6 @@ //! Database repositories pub mod alerts; -// TODO: Add threats and containers repositories +pub mod log_sources; pub use alerts::*; diff --git a/src/docker/client.rs b/src/docker/client.rs index 6bf03e3..751fe14 100644 --- a/src/docker/client.rs +++ b/src/docker/client.rs @@ -29,7 +29,7 @@ impl DockerClient { /// List all containers pub async fn list_containers(&self, all: bool) -> Result> { - let options = Some(ListContainersOptions { + let options: Option> = Some(ListContainersOptions { all, size: false, ..Default::default() @@ -85,7 +85,7 @@ impl DockerClient { /// Quarantine a container (disconnect from all networks) pub async fn quarantine_container(&self, container_id: &str) -> Result<()> { // List all networks - let networks: Vec = self.client + let networks: Vec = self.client .list_networks(None::>) .await .context("Failed to list networks")?; @@ -104,7 +104,7 @@ impl DockerClient { }; let _ = self.client - .disconnect_network(&name, Some(options)) + .disconnect_network(&name, options) .await; } } diff --git a/src/lib.rs b/src/lib.rs index 622b684..8a64c1d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,8 +46,7 @@ pub mod models; #[cfg(target_os = "linux")] pub mod firewall; -// Security modules - Collectors -#[cfg(target_os = "linux")] +// Security modules - Collectors (cross-platform; Linux-specific internals are gated within) pub mod collectors; // Optional modules @@ -56,10 +55,14 @@ pub mod response; pub mod correlator; pub mod baselines; pub mod database; +pub mod docker; // Configuration pub mod config; +// Log sniffing +pub mod sniff; + // Re-export commonly used types pub use events::syscall::{SyscallEvent, SyscallType}; pub use events::security::{SecurityEvent, NetworkEvent, ContainerEvent, AlertEvent}; @@ -74,7 +77,6 @@ pub use alerting::{NotificationChannel, NotificationConfig}; pub use firewall::{QuarantineManager, QuarantineState}; #[cfg(target_os = "linux")] pub use firewall::{ResponseAction, ResponseChain, ResponseExecutor, ResponseType}; -#[cfg(target_os = "linux")] pub use collectors::{EbpfLoader, SyscallMonitor}; // Rules diff --git a/src/main.rs b/src/main.rs index 33ccb20..4bb0619 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,26 +22,47 @@ mod config; mod api; mod database; mod docker; +mod events; +mod rules; +mod alerting; +mod models; +mod cli; +mod sniff; use std::{io, env}; use actix_web::{HttpServer, App, web}; use actix_cors::Cors; +use clap::Parser; use tracing::{Level, info}; use tracing_subscriber::FmtSubscriber; use database::{create_pool, init_database}; +use cli::{Cli, Command}; #[actix_rt::main] async fn main() -> io::Result<()> { // Load environment dotenv::dotenv().expect("Could not read .env file"); + // Parse CLI arguments + let cli = Cli::parse(); + // Setup logging - env::set_var("RUST_LOG", "stackdog=info,actix_web=info"); + // Only set default RUST_LOG if user hasn't configured it + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "stackdog=info,actix_web=info"); + } env_logger::init(); - // Setup tracing + // Setup tracing — respect RUST_LOG for level + let max_level = if env::var("RUST_LOG").map(|v| v.contains("debug")).unwrap_or(false) { + Level::DEBUG + } else if env::var("RUST_LOG").map(|v| v.contains("trace")).unwrap_or(false) { + Level::TRACE + } else { + Level::INFO + }; let subscriber = FmtSubscriber::builder() - .with_max_level(Level::INFO) + .with_max_level(max_level) .finish(); tracing::subscriber::set_global_default(subscriber) .expect("setting default subscriber failed"); @@ -49,8 +70,17 @@ async fn main() -> io::Result<()> { info!("🐕 Stackdog Security starting..."); info!("Platform: {}", std::env::consts::OS); info!("Architecture: {}", std::env::consts::ARCH); - - // Display configuration + + match cli.command { + Some(Command::Sniff { once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook }) => { + run_sniff(once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook).await + } + // Default: serve (backward compatible) + Some(Command::Serve) | None => run_serve().await, + } +} + +async fn run_serve() -> io::Result<()> { let app_host = env::var("APP_HOST").unwrap_or_else(|_| "0.0.0.0".to_string()); let app_port = env::var("APP_PORT").unwrap_or_else(|_| "5000".to_string()); let database_url = env::var("DATABASE_URL").unwrap_or_else(|_| "./stackdog.db".to_string()); @@ -78,6 +108,9 @@ async fn main() -> io::Result<()> { info!(" POST /api/containers/:id/quar - Quarantine container"); info!(" GET /api/threats - List threats"); info!(" GET /api/threats/statistics - Threat statistics"); + info!(" GET /api/logs/sources - List log sources"); + info!(" POST /api/logs/sources - Add log source"); + info!(" GET /api/logs/summaries - List AI summaries"); info!(" WS /ws - WebSocket for real-time updates"); info!(""); info!("Web Dashboard: http://{}:{}", app_host, app_port); @@ -99,3 +132,46 @@ async fn main() -> io::Result<()> { .run() .await } + +async fn run_sniff( + once: bool, + consume: bool, + output: String, + sources: Option, + interval: u64, + ai_provider: Option, + ai_model: Option, + ai_api_url: Option, + slack_webhook: Option, +) -> io::Result<()> { + let config = sniff::config::SniffConfig::from_env_and_args( + once, + consume, + &output, + sources.as_deref(), + interval, + ai_provider.as_deref(), + ai_model.as_deref(), + ai_api_url.as_deref(), + slack_webhook.as_deref(), + ); + + info!("🔍 Stackdog Sniff starting..."); + info!("Mode: {}", if config.once { "one-shot" } else { "continuous" }); + info!("Consume: {}", config.consume); + info!("Output: {}", config.output_dir.display()); + info!("Interval: {}s", config.interval_secs); + info!("AI Provider: {:?}", config.ai_provider); + info!("AI Model: {}", config.ai_model); + info!("AI API URL: {}", config.ai_api_url); + if config.slack_webhook.is_some() { + info!("Slack: configured ✓"); + } + + let orchestrator = sniff::SniffOrchestrator::new(config) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + orchestrator.run().await + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) +} + diff --git a/src/sniff/analyzer.rs b/src/sniff/analyzer.rs new file mode 100644 index 0000000..5eee30e --- /dev/null +++ b/src/sniff/analyzer.rs @@ -0,0 +1,639 @@ +//! AI-powered log analysis engine +//! +//! Provides log summarization and anomaly detection via two backends: +//! - OpenAI-compatible API (works with OpenAI, Ollama, vLLM, etc.) +//! - Local Candle inference (requires `ml` feature) + +use anyhow::{Result, Context}; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::sniff::reader::LogEntry; + +/// Summary produced by AI analysis of log entries +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogSummary { + pub source_id: String, + pub period_start: DateTime, + pub period_end: DateTime, + pub total_entries: usize, + pub summary_text: String, + pub error_count: usize, + pub warning_count: usize, + pub key_events: Vec, + pub anomalies: Vec, +} + +/// An anomaly detected in log entries +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogAnomaly { + pub description: String, + pub severity: AnomalySeverity, + pub sample_line: String, +} + +/// Severity of a detected anomaly +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum AnomalySeverity { + Low, + Medium, + High, + Critical, +} + +impl std::fmt::Display for AnomalySeverity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AnomalySeverity::Low => write!(f, "Low"), + AnomalySeverity::Medium => write!(f, "Medium"), + AnomalySeverity::High => write!(f, "High"), + AnomalySeverity::Critical => write!(f, "Critical"), + } + } +} + +/// Trait for AI-powered log analysis +#[async_trait] +pub trait LogAnalyzer: Send + Sync { + /// Summarize a batch of log entries + async fn summarize(&self, entries: &[LogEntry]) -> Result; +} + +/// OpenAI-compatible API backend (works with OpenAI, Ollama, vLLM, etc.) +pub struct OpenAiAnalyzer { + api_url: String, + api_key: Option, + model: String, + client: reqwest::Client, +} + +impl OpenAiAnalyzer { + pub fn new(api_url: String, api_key: Option, model: String) -> Self { + Self { + api_url, + api_key, + model, + client: reqwest::Client::new(), + } + } + + fn build_prompt(entries: &[LogEntry]) -> String { + let lines: Vec<&str> = entries.iter().map(|e| e.line.as_str()).collect(); + let log_block = lines.join("\n"); + + format!( + "Analyze these log entries and provide a JSON response with:\n\ + 1. \"summary\": A concise summary of what happened\n\ + 2. \"error_count\": Number of errors found\n\ + 3. \"warning_count\": Number of warnings found\n\ + 4. \"key_events\": Array of important events (max 5)\n\ + 5. \"anomalies\": Array of objects with \"description\", \"severity\" (Low/Medium/High/Critical), \"sample_line\"\n\n\ + Respond ONLY with valid JSON, no markdown.\n\n\ + Log entries:\n{}", log_block + ) + } +} + +/// Response structure from the LLM +#[derive(Debug, Deserialize)] +struct LlmAnalysis { + summary: Option, + error_count: Option, + warning_count: Option, + key_events: Option>, + anomalies: Option>, +} + +#[derive(Debug, Deserialize)] +struct LlmAnomaly { + description: Option, + severity: Option, + sample_line: Option, +} + +/// OpenAI chat completion response +#[derive(Debug, Deserialize)] +struct ChatCompletionResponse { + choices: Vec, +} + +#[derive(Debug, Deserialize)] +struct ChatChoice { + message: ChatMessage, +} + +#[derive(Debug, Deserialize, Serialize)] +struct ChatMessage { + role: String, + content: String, +} + +/// Extract JSON from LLM response, handling markdown fences, preamble text, etc. +fn extract_json(content: &str) -> &str { + let trimmed = content.trim(); + + // Try ```json ... ``` fence + if let Some(start) = trimmed.find("```json") { + let after_fence = &trimmed[start + 7..]; + if let Some(end) = after_fence.find("```") { + return after_fence[..end].trim(); + } + } + + // Try ``` ... ``` fence (no language tag) + if let Some(start) = trimmed.find("```") { + let after_fence = &trimmed[start + 3..]; + if let Some(end) = after_fence.find("```") { + return after_fence[..end].trim(); + } + } + + // Try to find raw JSON object + if let Some(start) = trimmed.find('{') { + if let Some(end) = trimmed.rfind('}') { + if end > start { + return &trimmed[start..=end]; + } + } + } + + trimmed +} + +/// Parse LLM severity string to enum +fn parse_severity(s: &str) -> AnomalySeverity { + match s.to_lowercase().as_str() { + "critical" => AnomalySeverity::Critical, + "high" => AnomalySeverity::High, + "medium" => AnomalySeverity::Medium, + _ => AnomalySeverity::Low, + } +} + +/// Parse the LLM JSON response into a LogSummary +fn parse_llm_response(source_id: &str, entries: &[LogEntry], raw_json: &str) -> Result { + log::debug!("Parsing LLM response ({} bytes) for source {}", raw_json.len(), source_id); + log::trace!("Raw LLM response:\n{}", raw_json); + + let analysis: LlmAnalysis = serde_json::from_str(raw_json) + .context(format!( + "Failed to parse LLM response as JSON. Response starts with: {}", + &raw_json[..raw_json.len().min(200)] + ))?; + + log::debug!( + "LLM analysis parsed — summary: {:?}, errors: {:?}, warnings: {:?}, anomalies: {}", + analysis.summary.as_deref().map(|s| &s[..s.len().min(80)]), + analysis.error_count, + analysis.warning_count, + analysis.anomalies.as_ref().map(|a| a.len()).unwrap_or(0), + ); + + let anomalies = analysis.anomalies.unwrap_or_default() + .into_iter() + .map(|a| LogAnomaly { + description: a.description.unwrap_or_default(), + severity: parse_severity(&a.severity.unwrap_or_default()), + sample_line: a.sample_line.unwrap_or_default(), + }) + .collect(); + + let (start, end) = entry_time_range(entries); + + Ok(LogSummary { + source_id: source_id.to_string(), + period_start: start, + period_end: end, + total_entries: entries.len(), + summary_text: analysis.summary.unwrap_or_else(|| "No summary available".into()), + error_count: analysis.error_count.unwrap_or(0), + warning_count: analysis.warning_count.unwrap_or(0), + key_events: analysis.key_events.unwrap_or_default(), + anomalies, + }) +} + +/// Compute time range from entries +fn entry_time_range(entries: &[LogEntry]) -> (DateTime, DateTime) { + if entries.is_empty() { + let now = Utc::now(); + return (now, now); + } + let start = entries.iter().map(|e| e.timestamp).min().unwrap_or_else(Utc::now); + let end = entries.iter().map(|e| e.timestamp).max().unwrap_or_else(Utc::now); + (start, end) +} + +#[async_trait] +impl LogAnalyzer for OpenAiAnalyzer { + async fn summarize(&self, entries: &[LogEntry]) -> Result { + if entries.is_empty() { + log::debug!("OpenAiAnalyzer: no entries to analyze, returning empty summary"); + return Ok(LogSummary { + source_id: String::new(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 0, + summary_text: "No log entries to analyze".into(), + error_count: 0, + warning_count: 0, + key_events: Vec::new(), + anomalies: Vec::new(), + }); + } + + let prompt = Self::build_prompt(entries); + let source_id = &entries[0].source_id; + + log::debug!( + "Sending {} entries to AI API (model: {}, url: {})", + entries.len(), self.model, self.api_url + ); + log::trace!("Prompt:\n{}", prompt); + + let request_body = serde_json::json!({ + "model": self.model, + "messages": [ + { + "role": "system", + "content": "You are a log analysis assistant. Analyze logs and return structured JSON." + }, + { + "role": "user", + "content": prompt + } + ], + "temperature": 0.1 + }); + + let url = format!("{}/chat/completions", self.api_url.trim_end_matches('/')); + log::debug!("POST {}", url); + + let mut req = self.client.post(&url) + .header("Content-Type", "application/json"); + + if let Some(ref key) = self.api_key { + log::debug!("Using API key: {}...{}", &key[..key.len().min(4)], &key[key.len().saturating_sub(4)..]); + req = req.header("Authorization", format!("Bearer {}", key)); + } else { + log::debug!("No API key configured (using keyless access)"); + } + + let response = req + .json(&request_body) + .send() + .await + .context("Failed to send request to AI API")?; + + let status = response.status(); + log::debug!("AI API response status: {}", status); + + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + log::debug!("AI API error body: {}", body); + anyhow::bail!("AI API returned status {}: {}", status, body); + } + + let raw_body = response.text().await + .context("Failed to read AI API response body")?; + log::debug!("AI API response body ({} bytes)", raw_body.len()); + log::trace!("AI API raw response:\n{}", raw_body); + + let completion: ChatCompletionResponse = serde_json::from_str(&raw_body) + .context("Failed to parse AI API response as ChatCompletion")?; + + let content = completion.choices + .first() + .map(|c| c.message.content.clone()) + .unwrap_or_default(); + + log::debug!("LLM content ({} chars): {}", content.len(), &content[..content.len().min(200)]); + + // Extract JSON from response — LLMs often wrap in markdown code fences + let json_str = extract_json(&content); + log::debug!("Extracted JSON ({} chars)", json_str.len()); + + parse_llm_response(source_id, entries, json_str) + } +} + +/// Fallback local analyzer that uses pattern matching (no AI required) +pub struct PatternAnalyzer; + +impl PatternAnalyzer { + pub fn new() -> Self { + Self + } + + fn count_pattern(entries: &[LogEntry], patterns: &[&str]) -> usize { + entries.iter().filter(|e| { + let lower = e.line.to_lowercase(); + patterns.iter().any(|p| lower.contains(p)) + }).count() + } +} + +#[async_trait] +impl LogAnalyzer for PatternAnalyzer { + async fn summarize(&self, entries: &[LogEntry]) -> Result { + if entries.is_empty() { + log::debug!("PatternAnalyzer: no entries to analyze"); + return Ok(LogSummary { + source_id: String::new(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 0, + summary_text: "No log entries to analyze".into(), + error_count: 0, + warning_count: 0, + key_events: Vec::new(), + anomalies: Vec::new(), + }); + } + + let source_id = &entries[0].source_id; + let error_count = Self::count_pattern(entries, &["error", "err", "fatal", "panic", "exception"]); + let warning_count = Self::count_pattern(entries, &["warn", "warning"]); + let (start, end) = entry_time_range(entries); + + log::debug!( + "PatternAnalyzer [{}]: {} entries, {} errors, {} warnings", + source_id, entries.len(), error_count, warning_count + ); + + let mut anomalies = Vec::new(); + + // Detect error spikes + if error_count > entries.len() / 4 { + log::debug!( + "Error spike detected: {} errors / {} entries (threshold: >25%)", + error_count, entries.len() + ); + if let Some(sample) = entries.iter().find(|e| e.line.to_lowercase().contains("error")) { + anomalies.push(LogAnomaly { + description: format!("High error rate: {} errors in {} entries", error_count, entries.len()), + severity: AnomalySeverity::High, + sample_line: sample.line.clone(), + }); + } + } + + let summary_text = format!( + "{} log entries analyzed. {} errors, {} warnings detected.", + entries.len(), error_count, warning_count + ); + + Ok(LogSummary { + source_id: source_id.clone(), + period_start: start, + period_end: end, + total_entries: entries.len(), + summary_text, + error_count, + warning_count, + key_events: Vec::new(), + anomalies, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn make_entries(lines: &[&str]) -> Vec { + lines.iter().map(|line| LogEntry { + source_id: "test-source".into(), + timestamp: Utc::now(), + line: line.to_string(), + metadata: HashMap::new(), + }).collect() + } + + #[test] + fn test_anomaly_severity_display() { + assert_eq!(AnomalySeverity::Low.to_string(), "Low"); + assert_eq!(AnomalySeverity::Critical.to_string(), "Critical"); + } + + #[test] + fn test_parse_severity() { + assert_eq!(parse_severity("critical"), AnomalySeverity::Critical); + assert_eq!(parse_severity("High"), AnomalySeverity::High); + assert_eq!(parse_severity("MEDIUM"), AnomalySeverity::Medium); + assert_eq!(parse_severity("low"), AnomalySeverity::Low); + assert_eq!(parse_severity("unknown"), AnomalySeverity::Low); + } + + #[test] + fn test_build_prompt_contains_log_lines() { + let entries = make_entries(&["line 1", "line 2"]); + let prompt = OpenAiAnalyzer::build_prompt(&entries); + assert!(prompt.contains("line 1")); + assert!(prompt.contains("line 2")); + assert!(prompt.contains("JSON")); + } + + #[test] + fn test_parse_llm_response_valid() { + let entries = make_entries(&["test line"]); + let json = r#"{ + "summary": "System running normally", + "error_count": 0, + "warning_count": 1, + "key_events": ["Service started"], + "anomalies": [] + }"#; + + let summary = parse_llm_response("src-1", &entries, json).unwrap(); + assert_eq!(summary.source_id, "src-1"); + assert_eq!(summary.summary_text, "System running normally"); + assert_eq!(summary.error_count, 0); + assert_eq!(summary.warning_count, 1); + assert_eq!(summary.key_events.len(), 1); + assert!(summary.anomalies.is_empty()); + } + + #[test] + fn test_parse_llm_response_with_anomalies() { + let entries = make_entries(&["error: disk full"]); + let json = r#"{ + "summary": "Disk issue detected", + "error_count": 1, + "warning_count": 0, + "key_events": ["Disk full"], + "anomalies": [ + { + "description": "Disk full errors detected", + "severity": "Critical", + "sample_line": "error: disk full" + } + ] + }"#; + + let summary = parse_llm_response("src-1", &entries, json).unwrap(); + assert_eq!(summary.anomalies.len(), 1); + assert_eq!(summary.anomalies[0].severity, AnomalySeverity::Critical); + assert!(summary.anomalies[0].description.contains("Disk full")); + } + + #[test] + fn test_parse_llm_response_partial_fields() { + let entries = make_entries(&["line"]); + let json = r#"{"summary": "Minimal response"}"#; + + let summary = parse_llm_response("src-1", &entries, json).unwrap(); + assert_eq!(summary.summary_text, "Minimal response"); + assert_eq!(summary.error_count, 0); + assert!(summary.anomalies.is_empty()); + } + + #[test] + fn test_parse_llm_response_invalid_json() { + let entries = make_entries(&["line"]); + let result = parse_llm_response("src-1", &entries, "not json"); + assert!(result.is_err()); + } + + #[test] + fn test_extract_json_plain() { + let input = r#"{"summary": "ok"}"#; + assert_eq!(extract_json(input), input); + } + + #[test] + fn test_extract_json_markdown_fence() { + let input = "```json\n{\"summary\": \"ok\"}\n```"; + assert_eq!(extract_json(input), r#"{"summary": "ok"}"#); + } + + #[test] + fn test_extract_json_plain_fence() { + let input = "```\n{\"summary\": \"ok\"}\n```"; + assert_eq!(extract_json(input), r#"{"summary": "ok"}"#); + } + + #[test] + fn test_extract_json_with_preamble() { + let input = "Here is the analysis:\n{\"summary\": \"ok\", \"error_count\": 0}"; + assert_eq!(extract_json(input), r#"{"summary": "ok", "error_count": 0}"#); + } + + #[test] + fn test_extract_json_with_trailing_text() { + let input = "Sure! {\"summary\": \"ok\"} Hope this helps!"; + assert_eq!(extract_json(input), r#"{"summary": "ok"}"#); + } + + #[test] + fn test_entry_time_range_empty() { + let (start, end) = entry_time_range(&[]); + assert!(end >= start); + } + + #[test] + fn test_entry_time_range_multiple() { + let mut entries = make_entries(&["a", "b"]); + entries[0].timestamp = Utc::now() - chrono::Duration::hours(1); + let (start, end) = entry_time_range(&entries); + assert!(end > start); + } + + #[tokio::test] + async fn test_pattern_analyzer_empty() { + let analyzer = PatternAnalyzer::new(); + let summary = analyzer.summarize(&[]).await.unwrap(); + assert_eq!(summary.total_entries, 0); + assert!(summary.summary_text.contains("No log entries")); + } + + #[tokio::test] + async fn test_pattern_analyzer_counts_errors() { + let analyzer = PatternAnalyzer::new(); + let entries = make_entries(&[ + "INFO: started", + "ERROR: connection refused", + "WARN: disk space low", + "ERROR: timeout", + ]); + let summary = analyzer.summarize(&entries).await.unwrap(); + assert_eq!(summary.total_entries, 4); + assert_eq!(summary.error_count, 2); + assert_eq!(summary.warning_count, 1); + } + + #[tokio::test] + async fn test_pattern_analyzer_detects_error_spike() { + let analyzer = PatternAnalyzer::new(); + let entries = make_entries(&[ + "ERROR: fail 1", + "ERROR: fail 2", + "ERROR: fail 3", + "INFO: ok", + ]); + let summary = analyzer.summarize(&entries).await.unwrap(); + assert!(!summary.anomalies.is_empty()); + assert_eq!(summary.anomalies[0].severity, AnomalySeverity::High); + } + + #[tokio::test] + async fn test_pattern_analyzer_no_anomaly_when_low_errors() { + let analyzer = PatternAnalyzer::new(); + let entries = make_entries(&[ + "INFO: all good", + "INFO: running fine", + "INFO: healthy", + "ERROR: one blip", + ]); + let summary = analyzer.summarize(&entries).await.unwrap(); + assert!(summary.anomalies.is_empty()); + } + + #[test] + fn test_openai_analyzer_new() { + let analyzer = OpenAiAnalyzer::new( + "http://localhost:11434/v1".into(), + None, + "llama3".into(), + ); + assert_eq!(analyzer.api_url, "http://localhost:11434/v1"); + assert!(analyzer.api_key.is_none()); + assert_eq!(analyzer.model, "llama3"); + } + + #[tokio::test] + async fn test_openai_analyzer_empty_entries() { + let analyzer = OpenAiAnalyzer::new( + "http://localhost:11434/v1".into(), + None, + "llama3".into(), + ); + let summary = analyzer.summarize(&[]).await.unwrap(); + assert_eq!(summary.total_entries, 0); + } + + #[test] + fn test_log_summary_serialization() { + let summary = LogSummary { + source_id: "test".into(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 10, + summary_text: "All good".into(), + error_count: 0, + warning_count: 0, + key_events: vec!["Started".into()], + anomalies: vec![LogAnomaly { + description: "Test anomaly".into(), + severity: AnomalySeverity::Medium, + sample_line: "WARN: something".into(), + }], + }; + let json = serde_json::to_string(&summary).unwrap(); + let deserialized: LogSummary = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.total_entries, 10); + assert_eq!(deserialized.anomalies[0].severity, AnomalySeverity::Medium); + } +} diff --git a/src/sniff/config.rs b/src/sniff/config.rs new file mode 100644 index 0000000..0fa0294 --- /dev/null +++ b/src/sniff/config.rs @@ -0,0 +1,311 @@ +//! Sniff configuration loaded from environment variables and CLI args + +use std::env; +use std::path::PathBuf; + +/// AI provider selection +#[derive(Debug, Clone, PartialEq)] +pub enum AiProvider { + /// OpenAI-compatible API (works with OpenAI, Ollama, vLLM, etc.) + OpenAi, + /// Local inference via Candle (requires `ml` feature) + Candle, +} + +impl AiProvider { + pub fn from_str(s: &str) -> Self { + match s.to_lowercase().as_str() { + "candle" => AiProvider::Candle, + // "ollama" uses the same OpenAI-compatible API client + "openai" | "ollama" => AiProvider::OpenAi, + _ => AiProvider::OpenAi, + } + } +} + +/// Configuration for the `stackdog sniff` command +#[derive(Debug, Clone)] +pub struct SniffConfig { + /// Run once then exit (vs continuous daemon mode) + pub once: bool, + /// Enable consume mode: archive + purge originals + pub consume: bool, + /// Output directory for archived/consumed logs + pub output_dir: PathBuf, + /// Additional log source paths (user-configured) + pub extra_sources: Vec, + /// Poll interval in seconds + pub interval_secs: u64, + /// AI provider to use for summarization + pub ai_provider: AiProvider, + /// AI API URL (for OpenAI-compatible providers) + pub ai_api_url: String, + /// AI API key (optional for local providers like Ollama) + pub ai_api_key: Option, + /// AI model name + pub ai_model: String, + /// Database URL + pub database_url: String, + /// Slack webhook URL for alert notifications + pub slack_webhook: Option, + /// Generic webhook URL for alert notifications + pub webhook_url: Option, +} + +impl SniffConfig { + /// Build config from environment variables, overridden by CLI args + pub fn from_env_and_args( + once: bool, + consume: bool, + output: &str, + sources: Option<&str>, + interval: u64, + ai_provider_arg: Option<&str>, + ai_model_arg: Option<&str>, + ai_api_url_arg: Option<&str>, + slack_webhook_arg: Option<&str>, + ) -> Self { + let env_sources = env::var("STACKDOG_LOG_SOURCES").unwrap_or_default(); + let mut extra_sources: Vec = env_sources + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + + if let Some(cli_sources) = sources { + for s in cli_sources.split(',') { + let trimmed = s.trim().to_string(); + if !trimmed.is_empty() && !extra_sources.contains(&trimmed) { + extra_sources.push(trimmed); + } + } + } + + let ai_provider_str = ai_provider_arg + .map(|s| s.to_string()) + .unwrap_or_else(|| env::var("STACKDOG_AI_PROVIDER").unwrap_or_else(|_| "openai".into())); + + let output_dir = if output != "./stackdog-logs/" { + PathBuf::from(output) + } else { + PathBuf::from( + env::var("STACKDOG_SNIFF_OUTPUT_DIR") + .unwrap_or_else(|_| output.to_string()), + ) + }; + + let interval_secs = if interval != 30 { + interval + } else { + env::var("STACKDOG_SNIFF_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(interval) + }; + + Self { + once, + consume, + output_dir, + extra_sources, + interval_secs, + ai_provider: AiProvider::from_str(&ai_provider_str), + ai_api_url: ai_api_url_arg + .map(|s| s.to_string()) + .or_else(|| env::var("STACKDOG_AI_API_URL").ok()) + .unwrap_or_else(|| "http://localhost:11434/v1".into()), + ai_api_key: env::var("STACKDOG_AI_API_KEY").ok(), + ai_model: ai_model_arg + .map(|s| s.to_string()) + .or_else(|| env::var("STACKDOG_AI_MODEL").ok()) + .unwrap_or_else(|| "llama3".into()), + database_url: env::var("DATABASE_URL") + .unwrap_or_else(|_| "./stackdog.db".into()), + slack_webhook: slack_webhook_arg + .map(|s| s.to_string()) + .or_else(|| env::var("STACKDOG_SLACK_WEBHOOK_URL").ok()), + webhook_url: env::var("STACKDOG_WEBHOOK_URL").ok(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + // Serialize env-mutating tests to avoid cross-contamination + static ENV_MUTEX: Mutex<()> = Mutex::new(()); + + fn clear_sniff_env() { + env::remove_var("STACKDOG_LOG_SOURCES"); + env::remove_var("STACKDOG_AI_PROVIDER"); + env::remove_var("STACKDOG_AI_API_URL"); + env::remove_var("STACKDOG_AI_API_KEY"); + env::remove_var("STACKDOG_AI_MODEL"); + env::remove_var("STACKDOG_SNIFF_OUTPUT_DIR"); + env::remove_var("STACKDOG_SNIFF_INTERVAL"); + env::remove_var("STACKDOG_SLACK_WEBHOOK_URL"); + env::remove_var("STACKDOG_WEBHOOK_URL"); + } + + #[test] + fn test_ai_provider_from_str() { + assert_eq!(AiProvider::from_str("openai"), AiProvider::OpenAi); + assert_eq!(AiProvider::from_str("OpenAI"), AiProvider::OpenAi); + assert_eq!(AiProvider::from_str("candle"), AiProvider::Candle); + assert_eq!(AiProvider::from_str("Candle"), AiProvider::Candle); + assert_eq!(AiProvider::from_str("unknown"), AiProvider::OpenAi); + } + + #[test] + fn test_sniff_config_defaults() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args(false, false, "./stackdog-logs/", None, 30, None, None, None, None); + assert!(!config.once); + assert!(!config.consume); + assert_eq!(config.output_dir, PathBuf::from("./stackdog-logs/")); + assert!(config.extra_sources.is_empty()); + assert_eq!(config.interval_secs, 30); + assert_eq!(config.ai_provider, AiProvider::OpenAi); + assert_eq!(config.ai_api_url, "http://localhost:11434/v1"); + assert!(config.ai_api_key.is_none()); + assert_eq!(config.ai_model, "llama3"); + } + + #[test] + fn test_sniff_config_cli_overrides() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args( + true, true, "/tmp/output/", Some("/var/log/app.log"), 60, Some("candle"), None, None, None, + ); + + assert!(config.once); + assert!(config.consume); + assert_eq!(config.output_dir, PathBuf::from("/tmp/output/")); + assert_eq!(config.extra_sources, vec!["/var/log/app.log"]); + assert_eq!(config.interval_secs, 60); + assert_eq!(config.ai_provider, AiProvider::Candle); + } + + #[test] + fn test_sniff_config_env_sources_merged_with_cli() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_LOG_SOURCES", "/var/log/syslog,/var/log/auth.log"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", Some("/var/log/app.log,/var/log/syslog"), 30, None, None, None, None, + ); + + assert!(config.extra_sources.contains(&"/var/log/syslog".to_string())); + assert!(config.extra_sources.contains(&"/var/log/auth.log".to_string())); + assert!(config.extra_sources.contains(&"/var/log/app.log".to_string())); + assert_eq!(config.extra_sources.len(), 3); + + clear_sniff_env(); + } + + #[test] + fn test_sniff_config_env_overrides_defaults() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_AI_API_URL", "https://api.openai.com/v1"); + env::set_var("STACKDOG_AI_API_KEY", "sk-test123"); + env::set_var("STACKDOG_AI_MODEL", "gpt-4o-mini"); + env::set_var("STACKDOG_SNIFF_INTERVAL", "45"); + env::set_var("STACKDOG_SNIFF_OUTPUT_DIR", "/data/logs/"); + + let config = SniffConfig::from_env_and_args(false, false, "./stackdog-logs/", None, 30, None, None, None, None); + assert_eq!(config.ai_api_url, "https://api.openai.com/v1"); + assert_eq!(config.ai_api_key, Some("sk-test123".into())); + assert_eq!(config.ai_model, "gpt-4o-mini"); + assert_eq!(config.interval_secs, 45); + assert_eq!(config.output_dir, PathBuf::from("/data/logs/")); + + clear_sniff_env(); + } + + #[test] + fn test_ollama_provider_alias() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + Some("ollama"), Some("qwen2.5-coder:latest"), None, None, + ); + // "ollama" maps to OpenAi internally (same API protocol) + assert_eq!(config.ai_provider, AiProvider::OpenAi); + assert_eq!(config.ai_model, "qwen2.5-coder:latest"); + assert_eq!(config.ai_api_url, "http://localhost:11434/v1"); + + clear_sniff_env(); + } + + #[test] + fn test_cli_args_override_env_vars() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_AI_MODEL", "gpt-4o-mini"); + env::set_var("STACKDOG_AI_API_URL", "https://api.openai.com/v1"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, Some("llama3"), Some("http://localhost:11434/v1"), None, + ); + // CLI args take priority over env vars + assert_eq!(config.ai_model, "llama3"); + assert_eq!(config.ai_api_url, "http://localhost:11434/v1"); + + clear_sniff_env(); + } + + #[test] + fn test_slack_webhook_from_cli() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, None, None, Some("https://hooks.slack.com/services/T/B/xxx"), + ); + assert_eq!(config.slack_webhook.as_deref(), Some("https://hooks.slack.com/services/T/B/xxx")); + + clear_sniff_env(); + } + + #[test] + fn test_slack_webhook_from_env() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T/B/env"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, None, None, None, + ); + assert_eq!(config.slack_webhook.as_deref(), Some("https://hooks.slack.com/services/T/B/env")); + + clear_sniff_env(); + } + + #[test] + fn test_slack_webhook_cli_overrides_env() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T/B/env"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, None, None, Some("https://hooks.slack.com/services/T/B/cli"), + ); + assert_eq!(config.slack_webhook.as_deref(), Some("https://hooks.slack.com/services/T/B/cli")); + + clear_sniff_env(); + } +} diff --git a/src/sniff/consumer.rs b/src/sniff/consumer.rs new file mode 100644 index 0000000..b594a63 --- /dev/null +++ b/src/sniff/consumer.rs @@ -0,0 +1,352 @@ +//! Log consumer: compress, deduplicate, and purge original logs +//! +//! When `--consume` is enabled, logs are archived to zstd-compressed files, +//! deduplicated, and then originals are purged to free disk space. + +use anyhow::{Result, Context}; +use chrono::Utc; +use std::collections::HashSet; +use std::collections::hash_map::DefaultHasher; +use std::fs::{self, File, OpenOptions}; +use std::hash::{Hash, Hasher}; +use std::io::{Write, BufWriter}; +use std::path::{Path, PathBuf}; + +use crate::sniff::reader::LogEntry; +use crate::sniff::discovery::LogSourceType; + +/// Result of a consume operation +#[derive(Debug, Clone, Default)] +pub struct ConsumeResult { + pub entries_archived: usize, + pub duplicates_skipped: usize, + pub bytes_freed: u64, + pub compressed_size: u64, +} + +/// Consumes log entries: deduplicates, compresses to zstd, and purges originals +pub struct LogConsumer { + output_dir: PathBuf, + seen_hashes: HashSet, + max_seen_hashes: usize, +} + +impl LogConsumer { + pub fn new(output_dir: PathBuf) -> Result { + fs::create_dir_all(&output_dir) + .with_context(|| format!("Failed to create output directory: {}", output_dir.display()))?; + + Ok(Self { + output_dir, + seen_hashes: HashSet::new(), + max_seen_hashes: 100_000, + }) + } + + /// Hash a log line for deduplication + fn hash_line(line: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + line.hash(&mut hasher); + hasher.finish() + } + + /// Deduplicate entries, returning only unique ones + pub fn deduplicate<'a>(&mut self, entries: &'a [LogEntry]) -> Vec<&'a LogEntry> { + // Evict oldest hashes if at capacity + if self.seen_hashes.len() > self.max_seen_hashes { + self.seen_hashes.clear(); + } + + let seen = &mut self.seen_hashes; + entries.iter().filter(|entry| { + let hash = Self::hash_line(&entry.line); + seen.insert(hash) + }).collect() + } + + /// Write entries to a zstd-compressed file + pub fn write_compressed(&self, entries: &[&LogEntry], source_name: &str) -> Result<(PathBuf, u64)> { + let timestamp = Utc::now().format("%Y%m%d_%H%M%S"); + let safe_name = source_name.replace(['/', '\\', ':', ' '], "_"); + let filename = format!("{}_{}.log.zst", safe_name, timestamp); + let path = self.output_dir.join(&filename); + + let file = File::create(&path) + .with_context(|| format!("Failed to create archive file: {}", path.display()))?; + + let encoder = zstd::Encoder::new(file, 3) + .context("Failed to create zstd encoder")?; + let mut writer = BufWriter::new(encoder); + + for entry in entries { + writeln!(writer, "{}\t{}", entry.timestamp.to_rfc3339(), entry.line)?; + } + + let encoder = writer.into_inner() + .map_err(|e| anyhow::anyhow!("Buffer flush error: {}", e))?; + encoder.finish() + .context("Failed to finish zstd encoding")?; + + let compressed_size = fs::metadata(&path)?.len(); + Ok((path, compressed_size)) + } + + /// Purge a file-based log source by truncating it + pub fn purge_file(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + + let original_size = fs::metadata(path)?.len(); + + // Truncate the file (preserves the fd for syslog daemons) + OpenOptions::new() + .write(true) + .truncate(true) + .open(path) + .with_context(|| format!("Failed to truncate log file: {}", path.display()))?; + + Ok(original_size) + } + + /// Purge Docker container logs by truncating the JSON log file + pub async fn purge_docker_logs(container_id: &str) -> Result { + // Docker stores logs at /var/lib/docker/containers//-json.log + let log_path = format!("/var/lib/docker/containers/{}/{}-json.log", container_id, container_id); + let path = Path::new(&log_path); + + if path.exists() { + Self::purge_file(path) + } else { + log::info!("Docker log file not found for container {}, skipping purge", container_id); + Ok(0) + } + } + + /// Full consume pipeline: deduplicate → compress → purge + pub async fn consume( + &mut self, + entries: &[LogEntry], + source_name: &str, + source_type: &LogSourceType, + source_path: &str, + ) -> Result { + if entries.is_empty() { + return Ok(ConsumeResult::default()); + } + + let total = entries.len(); + let unique_entries = self.deduplicate(entries); + let duplicates_skipped = total - unique_entries.len(); + + let (_, compressed_size) = self.write_compressed(&unique_entries, source_name)?; + + let bytes_freed = match source_type { + LogSourceType::DockerContainer => { + Self::purge_docker_logs(source_path).await? + } + LogSourceType::SystemLog | LogSourceType::CustomFile => { + let path = Path::new(source_path); + Self::purge_file(path)? + } + }; + + Ok(ConsumeResult { + entries_archived: unique_entries.len(), + duplicates_skipped, + bytes_freed, + compressed_size, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use std::collections::HashMap; + use std::io::Read; + + fn make_entry(line: &str) -> LogEntry { + LogEntry { + source_id: "test".into(), + timestamp: Utc::now(), + line: line.to_string(), + metadata: HashMap::new(), + } + } + + fn make_entries(lines: &[&str]) -> Vec { + lines.iter().map(|l| make_entry(l)).collect() + } + + #[test] + fn test_hash_line_deterministic() { + let h1 = LogConsumer::hash_line("hello world"); + let h2 = LogConsumer::hash_line("hello world"); + assert_eq!(h1, h2); + } + + #[test] + fn test_hash_line_different_for_different_inputs() { + let h1 = LogConsumer::hash_line("hello"); + let h2 = LogConsumer::hash_line("world"); + assert_ne!(h1, h2); + } + + #[test] + fn test_deduplicate_removes_duplicates() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["line A", "line B", "line A", "line C", "line B"]); + let unique = consumer.deduplicate(&entries); + assert_eq!(unique.len(), 3); + } + + #[test] + fn test_deduplicate_all_unique() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["line 1", "line 2", "line 3"]); + let unique = consumer.deduplicate(&entries); + assert_eq!(unique.len(), 3); + } + + #[test] + fn test_deduplicate_all_same() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["same", "same", "same"]); + let unique = consumer.deduplicate(&entries); + assert_eq!(unique.len(), 1); + } + + #[test] + fn test_write_compressed_creates_file() { + let dir = tempfile::tempdir().unwrap(); + let consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["line 1", "line 2"]); + let refs: Vec<&LogEntry> = entries.iter().collect(); + let (path, size) = consumer.write_compressed(&refs, "test-source").unwrap(); + + assert!(path.exists()); + assert!(size > 0); + assert!(path.to_string_lossy().ends_with(".log.zst")); + } + + #[test] + fn test_write_compressed_is_valid_zstd() { + let dir = tempfile::tempdir().unwrap(); + let consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["test line 1", "test line 2"]); + let refs: Vec<&LogEntry> = entries.iter().collect(); + let (path, _) = consumer.write_compressed(&refs, "zstd-test").unwrap(); + + // Decompress and verify + let file = File::open(&path).unwrap(); + let mut decoder = zstd::Decoder::new(file).unwrap(); + let mut content = String::new(); + decoder.read_to_string(&mut content).unwrap(); + + assert!(content.contains("test line 1")); + assert!(content.contains("test line 2")); + } + + #[test] + fn test_purge_file_truncates() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("to_purge.log"); + { + let mut f = File::create(&path).unwrap(); + write!(f, "lots of log data here that takes up space").unwrap(); + } + + let original_size = fs::metadata(&path).unwrap().len(); + assert!(original_size > 0); + + let freed = LogConsumer::purge_file(&path).unwrap(); + assert_eq!(freed, original_size); + + let new_size = fs::metadata(&path).unwrap().len(); + assert_eq!(new_size, 0); + } + + #[test] + fn test_purge_file_nonexistent() { + let freed = LogConsumer::purge_file(Path::new("/nonexistent/file.log")).unwrap(); + assert_eq!(freed, 0); + } + + #[tokio::test] + async fn test_consume_full_pipeline() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("app.log"); + { + let mut f = File::create(&log_path).unwrap(); + writeln!(f, "line 1").unwrap(); + writeln!(f, "line 2").unwrap(); + writeln!(f, "line 1").unwrap(); // duplicate + } + + let output_dir = dir.path().join("output"); + let mut consumer = LogConsumer::new(output_dir.clone()).unwrap(); + + let entries = make_entries(&["line 1", "line 2", "line 1"]); + let log_path_str = log_path.to_string_lossy().to_string(); + + let result = consumer.consume( + &entries, + "app", + &LogSourceType::CustomFile, + &log_path_str, + ).await.unwrap(); + + assert_eq!(result.entries_archived, 2); // deduplicated + assert_eq!(result.duplicates_skipped, 1); + assert!(result.compressed_size > 0); + assert!(result.bytes_freed > 0); + + // Original file should be truncated + let size = fs::metadata(&log_path).unwrap().len(); + assert_eq!(size, 0); + } + + #[tokio::test] + async fn test_consume_empty_entries() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let result = consumer.consume( + &[], + "empty", + &LogSourceType::SystemLog, + "/var/log/test", + ).await.unwrap(); + + assert_eq!(result.entries_archived, 0); + assert_eq!(result.duplicates_skipped, 0); + } + + #[test] + fn test_consumer_creates_output_dir() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("a/b/c"); + assert!(!nested.exists()); + + let consumer = LogConsumer::new(nested.clone()); + assert!(consumer.is_ok()); + assert!(nested.exists()); + } + + #[test] + fn test_consume_result_default() { + let result = ConsumeResult::default(); + assert_eq!(result.entries_archived, 0); + assert_eq!(result.bytes_freed, 0); + } +} diff --git a/src/sniff/discovery.rs b/src/sniff/discovery.rs new file mode 100644 index 0000000..c8acf92 --- /dev/null +++ b/src/sniff/discovery.rs @@ -0,0 +1,267 @@ +//! Log source discovery +//! +//! Scans for log sources across Docker containers, system log files, +//! and user-configured custom paths. + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +/// Type of log source +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum LogSourceType { + DockerContainer, + SystemLog, + CustomFile, +} + +impl std::fmt::Display for LogSourceType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LogSourceType::DockerContainer => write!(f, "DockerContainer"), + LogSourceType::SystemLog => write!(f, "SystemLog"), + LogSourceType::CustomFile => write!(f, "CustomFile"), + } + } +} + +impl LogSourceType { + pub fn from_str(s: &str) -> Self { + match s { + "DockerContainer" => LogSourceType::DockerContainer, + "SystemLog" => LogSourceType::SystemLog, + _ => LogSourceType::CustomFile, + } + } +} + +/// A discovered log source +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogSource { + pub id: String, + pub source_type: LogSourceType, + /// File path (for system/custom) or container ID (for Docker) + pub path_or_id: String, + pub name: String, + pub discovered_at: DateTime, + /// Byte offset for incremental reads (files only) + pub last_read_position: u64, +} + +impl LogSource { + pub fn new(source_type: LogSourceType, path_or_id: String, name: String) -> Self { + Self { + id: uuid::Uuid::new_v4().to_string(), + source_type, + path_or_id, + name, + discovered_at: Utc::now(), + last_read_position: 0, + } + } +} + +/// Well-known system log paths to probe +const SYSTEM_LOG_PATHS: &[&str] = &[ + "/var/log/syslog", + "/var/log/messages", + "/var/log/auth.log", + "/var/log/kern.log", + "/var/log/daemon.log", + "/var/log/secure", +]; + +/// Discover system log files that exist and are readable +pub fn discover_system_logs() -> Vec { + log::debug!("Probing {} system log paths", SYSTEM_LOG_PATHS.len()); + let sources: Vec = SYSTEM_LOG_PATHS + .iter() + .filter(|path| { + let exists = Path::new(path).exists(); + log::trace!("System log {} — exists: {}", path, exists); + exists + }) + .map(|path| { + let name = Path::new(path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + .to_string(); + LogSource::new(LogSourceType::SystemLog, path.to_string(), name) + }) + .collect(); + log::debug!("Discovered {} system log sources", sources.len()); + sources +} + +/// Register user-configured custom log file paths +pub fn discover_custom_sources(paths: &[String]) -> Vec { + log::debug!("Checking {} custom source paths", paths.len()); + paths + .iter() + .filter(|path| { + let exists = Path::new(path.as_str()).exists(); + if exists { + log::debug!("Custom source found: {}", path); + } else { + log::debug!("Custom source not found (skipped): {}", path); + } + exists + }) + .map(|path| { + let name = Path::new(path.as_str()) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("custom") + .to_string(); + LogSource::new(LogSourceType::CustomFile, path.clone(), name) + }) + .collect() +} + +/// Discover Docker container log sources +pub async fn discover_docker_sources() -> Result> { + use crate::docker::DockerClient; + + let client = match DockerClient::new().await { + Ok(c) => c, + Err(e) => { + log::warn!("Docker not available for log discovery: {}", e); + return Ok(Vec::new()); + } + }; + + let containers = client.list_containers(false).await?; + let sources = containers + .into_iter() + .map(|c| { + let name = format!("docker:{}", c.name); + LogSource::new(LogSourceType::DockerContainer, c.id, name) + }) + .collect(); + + Ok(sources) +} + +/// Run full discovery across all source types +pub async fn discover_all(extra_paths: &[String]) -> Result> { + let mut sources = Vec::new(); + + // System logs + let sys = discover_system_logs(); + log::debug!("System log discovery: {} sources", sys.len()); + sources.extend(sys); + + // Custom paths + let custom = discover_custom_sources(extra_paths); + log::debug!("Custom source discovery: {} sources", custom.len()); + sources.extend(custom); + + // Docker containers + match discover_docker_sources().await { + Ok(docker_sources) => { + log::debug!("Docker discovery: {} containers", docker_sources.len()); + sources.extend(docker_sources); + } + Err(e) => log::warn!("Docker discovery failed: {}", e), + } + + log::debug!("Total discovered sources: {}", sources.len()); + for s in &sources { + log::debug!(" [{:?}] {} — {}", s.source_type, s.name, s.path_or_id); + } + + Ok(sources) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_log_source_type_display() { + assert_eq!(LogSourceType::DockerContainer.to_string(), "DockerContainer"); + assert_eq!(LogSourceType::SystemLog.to_string(), "SystemLog"); + assert_eq!(LogSourceType::CustomFile.to_string(), "CustomFile"); + } + + #[test] + fn test_log_source_type_from_str() { + assert_eq!(LogSourceType::from_str("DockerContainer"), LogSourceType::DockerContainer); + assert_eq!(LogSourceType::from_str("SystemLog"), LogSourceType::SystemLog); + assert_eq!(LogSourceType::from_str("CustomFile"), LogSourceType::CustomFile); + assert_eq!(LogSourceType::from_str("anything"), LogSourceType::CustomFile); + } + + #[test] + fn test_log_source_new() { + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog".into(), + ); + assert_eq!(source.source_type, LogSourceType::SystemLog); + assert_eq!(source.path_or_id, "/var/log/syslog"); + assert_eq!(source.name, "syslog"); + assert_eq!(source.last_read_position, 0); + assert!(!source.id.is_empty()); + } + + #[test] + fn test_discover_custom_sources_existing_file() { + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "test log line").unwrap(); + let path = tmp.path().to_string_lossy().to_string(); + + let sources = discover_custom_sources(&[path.clone()]); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].source_type, LogSourceType::CustomFile); + assert_eq!(sources[0].path_or_id, path); + } + + #[test] + fn test_discover_custom_sources_nonexistent_file() { + let sources = discover_custom_sources(&["/nonexistent/path/log.txt".into()]); + assert!(sources.is_empty()); + } + + #[test] + fn test_discover_custom_sources_mixed() { + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "log").unwrap(); + let existing = tmp.path().to_string_lossy().to_string(); + + let sources = discover_custom_sources(&[ + existing.clone(), + "/does/not/exist.log".into(), + ]); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].path_or_id, existing); + } + + #[test] + fn test_discover_system_logs_returns_only_existing() { + let sources = discover_system_logs(); + for source in &sources { + assert_eq!(source.source_type, LogSourceType::SystemLog); + assert!(Path::new(&source.path_or_id).exists()); + } + } + + #[test] + fn test_log_source_serialization() { + let source = LogSource::new( + LogSourceType::DockerContainer, + "abc123def456".into(), + "docker:myapp".into(), + ); + let json = serde_json::to_string(&source).unwrap(); + let deserialized: LogSource = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.source_type, LogSourceType::DockerContainer); + assert_eq!(deserialized.path_or_id, "abc123def456"); + assert_eq!(deserialized.name, "docker:myapp"); + } +} diff --git a/src/sniff/mod.rs b/src/sniff/mod.rs new file mode 100644 index 0000000..4372bd2 --- /dev/null +++ b/src/sniff/mod.rs @@ -0,0 +1,268 @@ +//! Log sniffing module +//! +//! Discovers, reads, analyzes, and optionally consumes logs from +//! Docker containers, system log files, and custom sources. + +pub mod config; +pub mod discovery; +pub mod reader; +pub mod analyzer; +pub mod consumer; +pub mod reporter; + +use anyhow::Result; +use crate::database::connection::{create_pool, init_database, DbPool}; +use crate::alerting::notifications::NotificationConfig; +use crate::sniff::config::SniffConfig; +use crate::sniff::discovery::LogSourceType; +use crate::sniff::reader::{LogReader, FileLogReader, DockerLogReader}; +use crate::sniff::analyzer::{LogAnalyzer, PatternAnalyzer}; +use crate::sniff::consumer::LogConsumer; +use crate::sniff::reporter::Reporter; +use crate::database::repositories::log_sources as log_sources_repo; + +/// Main orchestrator for the sniff command +pub struct SniffOrchestrator { + config: SniffConfig, + pool: DbPool, + reporter: Reporter, +} + +impl SniffOrchestrator { + pub fn new(config: SniffConfig) -> Result { + let pool = create_pool(&config.database_url)?; + init_database(&pool)?; + + let mut notification_config = NotificationConfig::default(); + if let Some(ref url) = config.slack_webhook { + notification_config = notification_config.with_slack_webhook(url.clone()); + } + if let Some(ref url) = config.webhook_url { + notification_config = notification_config.with_webhook_url(url.clone()); + } + let reporter = Reporter::new(notification_config); + + Ok(Self { config, pool, reporter }) + } + + /// Create the appropriate AI analyzer based on config + fn create_analyzer(&self) -> Box { + match self.config.ai_provider { + config::AiProvider::OpenAi => { + log::debug!( + "Creating OpenAI-compatible analyzer (model: {}, url: {})", + self.config.ai_model, self.config.ai_api_url + ); + Box::new(analyzer::OpenAiAnalyzer::new( + self.config.ai_api_url.clone(), + self.config.ai_api_key.clone(), + self.config.ai_model.clone(), + )) + } + config::AiProvider::Candle => { + log::info!("Using pattern analyzer (Candle backend not yet implemented)"); + Box::new(PatternAnalyzer::new()) + } + } + } + + /// Build readers for discovered sources, restoring saved positions from DB + fn build_readers(&self, sources: &[discovery::LogSource]) -> Vec> { + sources.iter().filter_map(|source| { + let saved = log_sources_repo::get_log_source_by_path(&self.pool, &source.path_or_id) + .ok() + .flatten(); + let offset = saved.map(|s| s.last_read_position).unwrap_or(0); + + match source.source_type { + LogSourceType::SystemLog | LogSourceType::CustomFile => { + Some(Box::new(FileLogReader::new( + source.id.clone(), + source.path_or_id.clone(), + offset, + )) as Box) + } + LogSourceType::DockerContainer => { + Some(Box::new(DockerLogReader::new( + source.id.clone(), + source.path_or_id.clone(), + )) as Box) + } + } + }).collect() + } + + /// Run a single sniff pass: discover → read → analyze → report → consume + pub async fn run_once(&self) -> Result { + let mut result = SniffPassResult::default(); + + // 1. Discover sources + log::debug!("Step 1: discovering log sources..."); + let sources = discovery::discover_all(&self.config.extra_sources).await?; + result.sources_found = sources.len(); + log::debug!("Discovered {} sources", sources.len()); + + // Register sources in DB + for source in &sources { + let _ = log_sources_repo::upsert_log_source(&self.pool, source); + } + + // 2. Build readers and analyzer + log::debug!("Step 2: building readers and analyzer..."); + let mut readers = self.build_readers(&sources); + let analyzer = self.create_analyzer(); + let mut consumer = if self.config.consume { + log::debug!("Consume mode enabled, output: {}", self.config.output_dir.display()); + Some(LogConsumer::new(self.config.output_dir.clone())?) + } else { + None + }; + + // 3. Process each source + let reader_count = readers.len(); + for (i, reader) in readers.iter_mut().enumerate() { + log::debug!("Step 3: reading source {}/{} ({})", i + 1, reader_count, reader.source_id()); + let entries = reader.read_new_entries().await?; + if entries.is_empty() { + log::debug!(" No new entries, skipping"); + continue; + } + + result.total_entries += entries.len(); + log::debug!(" Read {} entries", entries.len()); + + // 4. Analyze + log::debug!("Step 4: analyzing {} entries...", entries.len()); + let summary = analyzer.summarize(&entries).await?; + log::debug!( + " Analysis complete: {} errors, {} warnings, {} anomalies", + summary.error_count, summary.warning_count, summary.anomalies.len() + ); + + // 5. Report + log::debug!("Step 5: reporting results..."); + let report = self.reporter.report(&summary, Some(&self.pool))?; + result.anomalies_found += report.anomalies_reported; + + // 6. Consume (if enabled) + if let Some(ref mut cons) = consumer { + if i < sources.len() { + log::debug!("Step 6: consuming entries..."); + let source = &sources[i]; + let consume_result = cons.consume( + &entries, + &source.name, + &source.source_type, + &source.path_or_id, + ).await?; + result.bytes_freed += consume_result.bytes_freed; + result.entries_archived += consume_result.entries_archived; + log::debug!(" Consumed: {} archived, {} bytes freed", + consume_result.entries_archived, consume_result.bytes_freed); + } + } + + // 7. Update read position + log::debug!("Step 7: saving read position ({})", reader.position()); + let _ = log_sources_repo::update_read_position( + &self.pool, + reader.source_id(), + reader.position(), + ); + } + + Ok(result) + } + + /// Run the sniff loop (continuous or one-shot) + pub async fn run(&self) -> Result<()> { + log::info!("🔍 Sniff orchestrator started"); + + loop { + match self.run_once().await { + Ok(result) => { + log::info!( + "Sniff pass: {} sources, {} entries, {} anomalies, {} bytes freed", + result.sources_found, + result.total_entries, + result.anomalies_found, + result.bytes_freed, + ); + } + Err(e) => { + log::error!("Sniff pass failed: {}", e); + } + } + + if self.config.once { + log::info!("🏁 One-shot mode: exiting after single pass"); + break; + } + + tokio::time::sleep(tokio::time::Duration::from_secs(self.config.interval_secs)).await; + } + + Ok(()) + } +} + +/// Result of a single sniff pass +#[derive(Debug, Clone, Default)] +pub struct SniffPassResult { + pub sources_found: usize, + pub total_entries: usize, + pub anomalies_found: usize, + pub bytes_freed: u64, + pub entries_archived: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sniff_pass_result_default() { + let result = SniffPassResult::default(); + assert_eq!(result.sources_found, 0); + assert_eq!(result.total_entries, 0); + assert_eq!(result.anomalies_found, 0); + assert_eq!(result.bytes_freed, 0); + } + + #[test] + fn test_orchestrator_creates_with_memory_db() { + let mut config = SniffConfig::from_env_and_args( + true, false, "./stackdog-logs/", None, 30, None, None, None, None, + ); + config.database_url = ":memory:".into(); + + let orchestrator = SniffOrchestrator::new(config); + assert!(orchestrator.is_ok()); + } + + #[tokio::test] + async fn test_orchestrator_run_once_with_file() { + use std::io::Write; + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("test.log"); + { + let mut f = std::fs::File::create(&log_path).unwrap(); + writeln!(f, "INFO: service started").unwrap(); + writeln!(f, "ERROR: connection failed").unwrap(); + writeln!(f, "WARN: retry in 5s").unwrap(); + } + + let mut config = SniffConfig::from_env_and_args( + true, false, "./stackdog-logs/", + Some(&log_path.to_string_lossy()), + 30, Some("candle"), None, None, None, + ); + config.database_url = ":memory:".into(); + + let orchestrator = SniffOrchestrator::new(config).unwrap(); + let result = orchestrator.run_once().await.unwrap(); + + assert!(result.sources_found >= 1); + assert!(result.total_entries >= 3); + } +} diff --git a/src/sniff/reader.rs b/src/sniff/reader.rs new file mode 100644 index 0000000..f97cabf --- /dev/null +++ b/src/sniff/reader.rs @@ -0,0 +1,423 @@ +//! Log readers for different source types +//! +//! Implements the `LogReader` trait for file-based logs, Docker container logs, +//! and systemd journal (Linux only). + +use anyhow::Result; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::fs::File; +use std::path::Path; + +/// A single log entry from any source +#[derive(Debug, Clone)] +pub struct LogEntry { + pub source_id: String, + pub timestamp: DateTime, + pub line: String, + pub metadata: HashMap, +} + +/// Trait for reading log entries from a source +#[async_trait] +pub trait LogReader: Send + Sync { + /// Read new entries since the last read position + async fn read_new_entries(&mut self) -> Result>; + /// Return the source identifier + fn source_id(&self) -> &str; + /// Return current read position (bytes for files, opaque for others) + fn position(&self) -> u64; +} + +/// Reads log entries from a regular file, tracking byte offset +pub struct FileLogReader { + source_id: String, + path: String, + offset: u64, +} + +impl FileLogReader { + pub fn new(source_id: String, path: String, start_offset: u64) -> Self { + Self { + source_id, + path, + offset: start_offset, + } + } + + fn read_lines_from_offset(&mut self) -> Result> { + let path = Path::new(&self.path); + if !path.exists() { + log::debug!("Log file does not exist: {}", self.path); + return Ok(Vec::new()); + } + + let file = File::open(path)?; + let file_len = file.metadata()?.len(); + log::debug!("Reading {} (size: {} bytes, offset: {})", self.path, file_len, self.offset); + + // Handle file truncation (log rotation) + if self.offset > file_len { + log::debug!("File truncated (rotation?), resetting offset from {} to 0", self.offset); + self.offset = 0; + } + + let mut reader = BufReader::new(file); + reader.seek(SeekFrom::Start(self.offset))?; + + let mut entries = Vec::new(); + let mut line = String::new(); + + while reader.read_line(&mut line)? > 0 { + let trimmed = line.trim_end().to_string(); + if !trimmed.is_empty() { + entries.push(LogEntry { + source_id: self.source_id.clone(), + timestamp: Utc::now(), + line: trimmed, + metadata: HashMap::from([ + ("source_path".into(), self.path.clone()), + ]), + }); + } + line.clear(); + } + + self.offset = reader.stream_position()?; + log::debug!("Read {} entries from {}, new offset: {}", entries.len(), self.path, self.offset); + Ok(entries) + } +} + +#[async_trait] +impl LogReader for FileLogReader { + async fn read_new_entries(&mut self) -> Result> { + self.read_lines_from_offset() + } + + fn source_id(&self) -> &str { + &self.source_id + } + + fn position(&self) -> u64 { + self.offset + } +} + +/// Reads logs from a Docker container via the bollard API +pub struct DockerLogReader { + source_id: String, + container_id: String, + last_timestamp: Option, +} + +impl DockerLogReader { + pub fn new(source_id: String, container_id: String) -> Self { + Self { + source_id, + container_id, + last_timestamp: None, + } + } +} + +#[async_trait] +impl LogReader for DockerLogReader { + async fn read_new_entries(&mut self) -> Result> { + use bollard::Docker; + use bollard::container::LogsOptions; + use futures_util::stream::StreamExt; + + let docker = match Docker::connect_with_local_defaults() { + Ok(d) => d, + Err(e) => { + log::warn!("Docker not available: {}", e); + return Ok(Vec::new()); + } + }; + + let options = LogsOptions:: { + stdout: true, + stderr: true, + since: self.last_timestamp.unwrap_or(0), + timestamps: true, + tail: if self.last_timestamp.is_none() { "100".to_string() } else { "all".to_string() }, + ..Default::default() + }; + + let mut stream = docker.logs(&self.container_id, Some(options)); + let mut entries = Vec::new(); + + while let Some(result) = stream.next().await { + match result { + Ok(output) => { + let line = output.to_string(); + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() { + entries.push(LogEntry { + source_id: self.source_id.clone(), + timestamp: Utc::now(), + line: trimmed, + metadata: HashMap::from([ + ("container_id".into(), self.container_id.clone()), + ]), + }); + } + } + Err(e) => { + log::warn!("Error reading Docker logs for {}: {}", self.container_id, e); + break; + } + } + } + + self.last_timestamp = Some(Utc::now().timestamp()); + Ok(entries) + } + + fn source_id(&self) -> &str { + &self.source_id + } + + fn position(&self) -> u64 { + self.last_timestamp.unwrap_or(0) as u64 + } +} + +/// Reads logs from systemd journal (Linux only) +#[cfg(target_os = "linux")] +pub struct JournaldReader { + source_id: String, + cursor: Option, +} + +#[cfg(target_os = "linux")] +impl JournaldReader { + pub fn new(source_id: String) -> Self { + Self { + source_id, + cursor: None, + } + } +} + +#[cfg(target_os = "linux")] +#[async_trait] +impl LogReader for JournaldReader { + async fn read_new_entries(&mut self) -> Result> { + use tokio::process::Command; + + let mut cmd = Command::new("journalctl"); + cmd.arg("--no-pager") + .arg("-o").arg("short-iso") + .arg("-n").arg("200"); + + if let Some(ref cursor) = self.cursor { + cmd.arg("--after-cursor").arg(cursor); + } + + cmd.arg("--show-cursor"); + + let output = cmd.output().await?; + let stdout = String::from_utf8_lossy(&output.stdout); + let mut entries = Vec::new(); + + for line in stdout.lines() { + if line.starts_with("-- cursor:") { + self.cursor = line.strip_prefix("-- cursor: ").map(|s| s.to_string()); + continue; + } + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() { + entries.push(LogEntry { + source_id: self.source_id.clone(), + timestamp: Utc::now(), + line: trimmed, + metadata: HashMap::from([ + ("source".into(), "journald".into()), + ]), + }); + } + } + + Ok(entries) + } + + fn source_id(&self) -> &str { + &self.source_id + } + + fn position(&self) -> u64 { + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_log_entry_creation() { + let entry = LogEntry { + source_id: "test-source".into(), + timestamp: Utc::now(), + line: "Error: something went wrong".into(), + metadata: HashMap::from([("key".into(), "value".into())]), + }; + assert_eq!(entry.source_id, "test-source"); + assert!(entry.line.contains("Error")); + assert_eq!(entry.metadata.get("key"), Some(&"value".to_string())); + } + + #[test] + fn test_file_log_reader_new() { + let reader = FileLogReader::new("src-1".into(), "/tmp/test.log".into(), 0); + assert_eq!(reader.source_id(), "src-1"); + assert_eq!(reader.position(), 0); + } + + #[tokio::test] + async fn test_file_log_reader_reads_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "line 1").unwrap(); + writeln!(f, "line 2").unwrap(); + writeln!(f, "line 3").unwrap(); + } + + let mut reader = FileLogReader::new( + "test".into(), + path.to_string_lossy().to_string(), + 0, + ); + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].line, "line 1"); + assert_eq!(entries[1].line, "line 2"); + assert_eq!(entries[2].line, "line 3"); + } + + #[tokio::test] + async fn test_file_log_reader_incremental_reads() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("incremental.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "line A").unwrap(); + writeln!(f, "line B").unwrap(); + } + + let path_str = path.to_string_lossy().to_string(); + let mut reader = FileLogReader::new("inc".into(), path_str, 0); + + // First read + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 2); + + // No new lines → empty + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 0); + + // Append new lines + { + let mut f = std::fs::OpenOptions::new().append(true).open(&path).unwrap(); + writeln!(f, "line C").unwrap(); + } + + // Should only get the new line + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].line, "line C"); + } + + #[tokio::test] + async fn test_file_log_reader_handles_truncation() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("rotating.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "original long line with lots of content here").unwrap(); + } + + let path_str = path.to_string_lossy().to_string(); + let mut reader = FileLogReader::new("rot".into(), path_str, 0); + + // Read past original content + reader.read_new_entries().await.unwrap(); + let saved_pos = reader.position(); + assert!(saved_pos > 0); + + // Simulate log rotation: truncate and write shorter content + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "new").unwrap(); + } + + // Should detect truncation and read from beginning + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].line, "new"); + } + + #[tokio::test] + async fn test_file_log_reader_nonexistent_file() { + let mut reader = FileLogReader::new("missing".into(), "/nonexistent/file.log".into(), 0); + let entries = reader.read_new_entries().await.unwrap(); + assert!(entries.is_empty()); + } + + #[tokio::test] + async fn test_file_log_reader_skips_empty_lines() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("empty_lines.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "line 1").unwrap(); + writeln!(f).unwrap(); // empty line + writeln!(f, "line 3").unwrap(); + } + + let mut reader = FileLogReader::new( + "empty".into(), + path.to_string_lossy().to_string(), + 0, + ); + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].line, "line 1"); + assert_eq!(entries[1].line, "line 3"); + } + + #[tokio::test] + async fn test_file_log_reader_metadata_contains_path() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("meta.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "test").unwrap(); + } + + let path_str = path.to_string_lossy().to_string(); + let mut reader = FileLogReader::new("meta".into(), path_str.clone(), 0); + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries[0].metadata.get("source_path"), Some(&path_str)); + } + + #[test] + fn test_docker_log_reader_new() { + let reader = DockerLogReader::new("d-1".into(), "abc123".into()); + assert_eq!(reader.source_id(), "d-1"); + assert_eq!(reader.position(), 0); + } + + #[test] + fn test_file_log_reader_with_start_offset() { + let reader = FileLogReader::new("off".into(), "/tmp/test.log".into(), 1024); + assert_eq!(reader.position(), 1024); + } +} diff --git a/src/sniff/reporter.rs b/src/sniff/reporter.rs new file mode 100644 index 0000000..bfc3b55 --- /dev/null +++ b/src/sniff/reporter.rs @@ -0,0 +1,209 @@ +//! Log analysis reporter +//! +//! Converts log summaries and anomalies into alerts, then dispatches +//! them via the existing notification channels. + +use anyhow::Result; +use crate::alerting::alert::{Alert, AlertSeverity, AlertType}; +use crate::alerting::notifications::{NotificationChannel, NotificationConfig, route_by_severity}; +use crate::sniff::analyzer::{LogSummary, LogAnomaly, AnomalySeverity}; +use crate::database::connection::DbPool; +use crate::database::repositories::log_sources; + +/// Reports log analysis results to alert channels and persists summaries +pub struct Reporter { + notification_config: NotificationConfig, +} + +impl Reporter { + pub fn new(notification_config: NotificationConfig) -> Self { + Self { notification_config } + } + + /// Map anomaly severity to alert severity + fn map_severity(anomaly_severity: &AnomalySeverity) -> AlertSeverity { + match anomaly_severity { + AnomalySeverity::Low => AlertSeverity::Low, + AnomalySeverity::Medium => AlertSeverity::Medium, + AnomalySeverity::High => AlertSeverity::High, + AnomalySeverity::Critical => AlertSeverity::Critical, + } + } + + /// Report a log summary: persist to DB and send anomaly alerts + pub fn report(&self, summary: &LogSummary, pool: Option<&DbPool>) -> Result { + let mut alerts_sent = 0; + + // Persist summary to database + if let Some(pool) = pool { + log::debug!("Persisting summary for source {} to database", summary.source_id); + let _ = log_sources::create_log_summary( + pool, + &summary.source_id, + &summary.summary_text, + &summary.period_start.to_rfc3339(), + &summary.period_end.to_rfc3339(), + summary.total_entries as i64, + summary.error_count as i64, + summary.warning_count as i64, + ); + } + + // Generate alerts for anomalies + for anomaly in &summary.anomalies { + let alert_severity = Self::map_severity(&anomaly.severity); + + log::debug!( + "Generating alert: severity={}, description={}", + anomaly.severity, anomaly.description + ); + + let alert = Alert::new( + AlertType::AnomalyDetected, + alert_severity, + format!( + "[Log Sniff] {} — Source: {} | Sample: {}", + anomaly.description, summary.source_id, anomaly.sample_line + ), + ); + + // Route to appropriate notification channels + let channels = route_by_severity(alert_severity); + log::debug!("Routing alert to {} notification channels", channels.len()); + for channel in &channels { + match channel.send(&alert, &self.notification_config) { + Ok(_) => alerts_sent += 1, + Err(e) => log::warn!("Failed to send notification: {}", e), + } + } + } + + // Log summary to console + log::info!( + "📊 Log Summary [{}]: {} entries, {} errors, {} warnings, {} anomalies", + summary.source_id, + summary.total_entries, + summary.error_count, + summary.warning_count, + summary.anomalies.len(), + ); + + Ok(ReportResult { + anomalies_reported: summary.anomalies.len(), + notifications_sent: alerts_sent, + summary_persisted: pool.is_some(), + }) + } +} + +/// Result of a report operation +#[derive(Debug, Clone, Default)] +pub struct ReportResult { + pub anomalies_reported: usize, + pub notifications_sent: usize, + pub summary_persisted: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use crate::database::connection::{create_pool, init_database}; + + fn make_summary(anomalies: Vec) -> LogSummary { + LogSummary { + source_id: "test-source".into(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 100, + summary_text: "Test summary".into(), + error_count: 5, + warning_count: 3, + key_events: vec!["Service restarted".into()], + anomalies, + } + } + + #[test] + fn test_map_severity() { + assert_eq!(Reporter::map_severity(&AnomalySeverity::Low), AlertSeverity::Low); + assert_eq!(Reporter::map_severity(&AnomalySeverity::Medium), AlertSeverity::Medium); + assert_eq!(Reporter::map_severity(&AnomalySeverity::High), AlertSeverity::High); + assert_eq!(Reporter::map_severity(&AnomalySeverity::Critical), AlertSeverity::Critical); + } + + #[test] + fn test_report_no_anomalies() { + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![]); + let result = reporter.report(&summary, None).unwrap(); + assert_eq!(result.anomalies_reported, 0); + assert_eq!(result.notifications_sent, 0); + assert!(!result.summary_persisted); + } + + #[test] + fn test_report_with_anomalies_sends_alerts() { + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![ + LogAnomaly { + description: "High error rate".into(), + severity: AnomalySeverity::High, + sample_line: "ERROR: connection failed".into(), + }, + ]); + + let result = reporter.report(&summary, None).unwrap(); + assert_eq!(result.anomalies_reported, 1); + // Console channel is always available, so at least 1 notification sent + assert!(result.notifications_sent >= 1); + } + + #[test] + fn test_report_persists_to_database() { + let pool = create_pool(":memory:").unwrap(); + init_database(&pool).unwrap(); + + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![]); + + let result = reporter.report(&summary, Some(&pool)).unwrap(); + assert!(result.summary_persisted); + + // Verify summary was stored + let summaries = log_sources::list_summaries_for_source(&pool, "test-source").unwrap(); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].total_entries, 100); + } + + #[test] + fn test_report_multiple_anomalies() { + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![ + LogAnomaly { + description: "Error spike".into(), + severity: AnomalySeverity::Critical, + sample_line: "FATAL: OOM".into(), + }, + LogAnomaly { + description: "Unusual pattern".into(), + severity: AnomalySeverity::Low, + sample_line: "DEBUG: retry".into(), + }, + ]); + + let result = reporter.report(&summary, None).unwrap(); + assert_eq!(result.anomalies_reported, 2); + assert!(result.notifications_sent >= 2); + } + + #[test] + fn test_reporter_new() { + let config = NotificationConfig::default(); + let reporter = Reporter::new(config); + // Just ensure it constructs without error + let summary = make_summary(vec![]); + let result = reporter.report(&summary, None); + assert!(result.is_ok()); + } +} diff --git a/tests/structure/mod_test.rs b/tests/structure/mod_test.rs index 4893d6a..ec4ea2b 100644 --- a/tests/structure/mod_test.rs +++ b/tests/structure/mod_test.rs @@ -5,64 +5,61 @@ #[test] fn test_collectors_module_imports() { - // Verify collectors module exists and can be imported - // This test will compile only if the module structure is correct - use crate::collectors; - - // Suppress unused import warning + use stackdog::collectors; let _ = std::marker::PhantomData::; } #[test] fn test_events_module_imports() { - use crate::events; + use stackdog::events; let _ = std::marker::PhantomData::; } #[test] fn test_rules_module_imports() { - use crate::rules; + use stackdog::rules; let _ = std::marker::PhantomData::; } #[test] fn test_ml_module_imports() { - use crate::ml; + use stackdog::ml; let _ = std::marker::PhantomData::; } +#[cfg(target_os = "linux")] #[test] fn test_firewall_module_imports() { - use crate::firewall; + use stackdog::firewall; let _ = std::marker::PhantomData::; } #[test] fn test_response_module_imports() { - use crate::response; + use stackdog::response; let _ = std::marker::PhantomData::; } #[test] fn test_correlator_module_imports() { - use crate::correlator; + use stackdog::correlator; let _ = std::marker::PhantomData::; } #[test] fn test_alerting_module_imports() { - use crate::alerting; + use stackdog::alerting; let _ = std::marker::PhantomData::; } #[test] fn test_baselines_module_imports() { - use crate::baselines; + use stackdog::baselines; let _ = std::marker::PhantomData::; } #[test] fn test_database_module_imports() { - use crate::database; + use stackdog::database; let _ = std::marker::PhantomData::; }