diff --git a/.env.sample b/.env.sample index 59074d1..17a893d 100644 --- a/.env.sample +++ b/.env.sample @@ -5,3 +5,21 @@ APP_HOST=0.0.0.0 APP_PORT=5000 DATABASE_URL=stackdog.db RUST_BACKTRACE=full + +# Log Sniff Configuration +#STACKDOG_LOG_SOURCES=/var/log/syslog,/var/log/auth.log +#STACKDOG_SNIFF_INTERVAL=30 +#STACKDOG_SNIFF_OUTPUT_DIR=./stackdog-logs/ + +# AI Provider Configuration +# Supports OpenAI, Ollama (http://localhost:11434/v1), or any OpenAI-compatible API +#STACKDOG_AI_PROVIDER=openai +#STACKDOG_AI_API_URL=http://localhost:11434/v1 +#STACKDOG_AI_API_KEY= +#STACKDOG_AI_MODEL=llama3 + +# Notification Channels +# Slack: create an incoming webhook at https://api.slack.com/messaging/webhooks +#STACKDOG_SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T.../B.../xxxxx +# Generic webhook endpoint for alert notifications +#STACKDOG_WEBHOOK_URL=https://example.com/webhook diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..2b679aa --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,113 @@ +# Stackdog Security — Copilot Instructions + +## What This Project Is + +Stackdog is a Rust-based security platform for Docker containers and Linux servers. It collects events via eBPF syscall monitoring, runs them through a rule/signature engine and optional ML anomaly detection, manages firewall responses (nftables/iptables + container quarantine), and exposes a REST + WebSocket API consumed by a React/TypeScript dashboard. + +## Workspace Structure + +This is a Cargo workspace with two crates: +- `.` — Main crate (`stackdog`): HTTP server, all security logic +- `ebpf/` — Separate crate (`stackdog-ebpf`): eBPF programs compiled for the kernel (uses `aya-ebpf`) + +## Build, Test, and Lint Commands + +```bash +# Build +cargo build +cargo build --release + +# Tests +cargo test --lib # Unit tests only (in-source) +cargo test --all # All tests including integration +cargo test --lib -- events:: # Run tests for a specific module +cargo test --lib -- rules::scorer # Run a single test by name prefix + +# Code quality +cargo fmt --all +cargo clippy --all +cargo audit # Dependency vulnerability scan + +# Benchmarks +cargo bench + +# Frontend (in web/) +npm test +npm run lint +npm run build +``` + +## Environment Setup + +Requires a `.env` file (copy `.env.sample`). Key variables: +``` +APP_HOST=0.0.0.0 +APP_PORT=5000 +DATABASE_URL=stackdog.db +RUST_BACKTRACE=full +``` + +System dependencies (Linux): `libsqlite3-dev libssl-dev clang llvm pkg-config` + +## Architecture + +``` +Collectors (Linux only) Rule Engine Response + eBPF syscall events → Signatures → nftables/iptables + Docker daemon events → Threat scoring → Container quarantine + Network events → ML anomaly det. → Alerting + + REST + WebSocket API + React/TypeScript UI +``` + +**Key src/ modules:** + +| Module | Purpose | +|---|---| +| `events/` | Core event types: `SyscallEvent`, `SecurityEvent`, `NetworkEvent`, `ContainerEvent` | +| `rules/` | Rule engine, signature database, threat scorer | +| `alerting/` | `AlertManager`, notification channels (Slack/email/webhook) | +| `collectors/` | eBPF loader, Docker daemon events, network collector (Linux only) | +| `firewall/` | nftables management, iptables fallback, `QuarantineManager` (Linux only) | +| `ml/` | Candle-based anomaly detection (optional `ml` feature) | +| `correlator/` | Event correlation engine | +| `baselines/` | Baseline learning for anomaly detection | +| `database/` | SQLite connection pool (`r2d2` + raw `rusqlite`), repositories | +| `api/` | actix-web REST endpoints + WebSocket | +| `response/` | Automated response action pipeline | + +## Key Conventions + +### Platform-Gating +Linux-only modules (`collectors`, `firewall`) and deps (aya, netlink) are gated: +```rust +#[cfg(target_os = "linux")] +pub mod firewall; +``` +The `ebpf` and `ml` features are opt-in and must be enabled explicitly: +```bash +cargo build --features ebpf +cargo build --features ml +``` + +### Error Handling +- Use `anyhow::{Result, Context}` for application/binary code +- Use `thiserror` for library error types +- Never use `.unwrap()` in production code; use `?` with `.context("...")` + +### Database +The project uses raw `rusqlite` with `r2d2` connection pooling. `DbPool` is `r2d2::Pool`. Tables are created with `CREATE TABLE IF NOT EXISTS` in `database::connection::init_database`. Repositories are in `src/database/repositories/` and receive a `&DbPool`. + +### API Routes +Each API sub-module exports a `configure_routes(cfg: &mut web::ServiceConfig)` function. All routes are composed in `api::configure_all_routes`, which is the single call site in `main.rs`. + +### Test Location +- **Unit tests**: `#[cfg(test)] mod tests { ... }` inside source files +- **Integration tests**: `tests/` directory at workspace root + +### eBPF Programs +The `ebpf/` crate is compiled separately for the Linux kernel. User-space loading is handled by `src/collectors/ebpf/` using the `aya` library. Kernel-side programs use `aya-ebpf`. + +### Async Runtime +The main binary uses `#[actix_rt::main]`. Library code uses `tokio`. Avoid mixing runtimes. diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d3a0eac..7917cda 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,7 +12,8 @@ on: jobs: cicd-linux-docker: name: Cargo and npm build - runs-on: ubuntu-latest + #runs-on: ubuntu-latest + runs-on: [self-hosted, linux] steps: - name: Checkout sources uses: actions/checkout@v2 @@ -135,7 +136,8 @@ jobs: cicd-docker: name: CICD Docker - runs-on: ubuntu-latest + #runs-on: ubuntu-latest + runs-on: [self-hosted, linux] needs: cicd-linux-docker steps: - name: Download app archive diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f15bf4c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,77 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + artifact: stackdog-linux-x86_64 + - target: aarch64-unknown-linux-gnu + artifact: stackdog-linux-aarch64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross + run: cargo install cross --git https://github.com/cross-rs/cross + + - name: Build release binary + run: cross build --release --target ${{ matrix.target }} + + - name: Package + run: | + mkdir -p dist + cp target/${{ matrix.target }}/release/stackdog dist/stackdog + cd dist + tar czf ${{ matrix.artifact }}.tar.gz stackdog + sha256sum ${{ matrix.artifact }}.tar.gz > ${{ matrix.artifact }}.tar.gz.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: | + dist/${{ matrix.artifact }}.tar.gz + dist/${{ matrix.artifact }}.tar.gz.sha256 + + release: + name: Create GitHub Release + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Create release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: | + artifacts/*.tar.gz + artifacts/*.sha256 diff --git a/.gitignore b/.gitignore index 49c5b70..2b846cb 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,4 @@ Cargo.lock # End of https://www.gitignore.io/api/rust,code .idea -<<<<<<< HEAD -======= -*.db ->>>>>>> testing +docs/tasks/ diff --git a/.qwen/PROJECT_MEMORY.md b/.qwen/PROJECT_MEMORY.md new file mode 100644 index 0000000..61d707c --- /dev/null +++ b/.qwen/PROJECT_MEMORY.md @@ -0,0 +1,277 @@ +# Stackdog Security - Project Memory + +## Project Identity + +**Name:** Stackdog Security +**Version:** 0.1.0 (Security-focused rewrite) +**Type:** Container and Linux Server Security Platform +**License:** MIT + +## Core Mission + +> Provide real-time security monitoring, AI-powered threat detection, and automated response for Docker containers and Linux servers using Rust and eBPF technologies. + +## Key Decisions + +### Architecture Decisions + +| ID | Decision | Rationale | Date | +|----|----------|-----------|------| +| **ARCH-001** | Use eBPF for syscall monitoring | Minimal overhead (<5% CPU), kernel-level visibility, safe (sandboxed) | 2026-03-13 | +| **ARCH-002** | Use Candle for ML instead of Python | Native Rust, no Python dependencies, fast inference, maintained by HuggingFace | 2026-03-13 | +| **ARCH-003** | Use nftables over iptables | Modern, faster, better batch support, iptables as fallback | 2026-03-13 | +| **ARCH-004** | TDD development methodology | Better code quality, maintainability, regression prevention | 2026-03-13 | +| **ARCH-005** | Functional programming principles | Immutability, fewer bugs, easier reasoning about code | 2026-03-13 | + +### Technology Choices + +| Component | Technology | Alternatives Considered | +|-----------|-----------|------------------------| +| **eBPF Framework** | aya-rs | libbpf (C), bcc (Python) | +| **ML Framework** | Candle (HuggingFace) | PyTorch (Python), ONNX Runtime, linfa | +| **Web Framework** | Actix-web 4.x | Axum, Rocket | +| **Database** | SQLite + rusqlite + r2d2 | PostgreSQL, Redis | +| **Firewall** | nftables (netlink) | iptables, firewalld | + +## Project Structure + +``` +stackdog/ +├── src/ +│ ├── collectors/ # Event collection (eBPF, Docker, etc.) +│ ├── events/ # Event types and structures +│ ├── ml/ # ML engine (Candle-based) +│ ├── firewall/ # Firewall management (nftables/iptables) +│ ├── response/ # Automated response actions +│ ├── correlator/ # Event correlation +│ ├── alerting/ # Alert system +│ ├── api/ # REST API + WebSocket +│ ├── config/ # Configuration +│ ├── models/ # Data models +│ ├── database/ # Database operations +│ └── utils/ # Utilities +├── ebpf/ # eBPF programs (separate crate) +├── web/ # React/TypeScript frontend +├── tests/ # Integration tests +├── benches/ # Performance benchmarks +└── models/ # Pre-trained ML models +``` + +## Development Principles + +### Clean Code (Robert C. Martin) + +1. **DRY** - Don't Repeat Yourself +2. **SRP** - Single Responsibility Principle +3. **OCP** - Open/Closed Principle +4. **DIP** - Dependency Inversion Principle +5. **Functional First** - Immutability, From/Into traits, builder pattern + +### TDD Workflow + +``` +Red → Green → Refactor +``` + +1. Write failing test +2. Run test (verify failure) +3. Implement minimal code to pass +4. Run test (verify pass) +5. Refactor (maintain passing tests) + +### Code Review Checklist + +- [ ] Tests written first (TDD) +- [ ] All tests pass +- [ ] Code formatted (`cargo fmt`) +- [ ] No clippy warnings +- [ ] DRY principle followed +- [ ] Functions < 50 lines +- [ ] Error handling comprehensive +- [ ] Documentation for public APIs + +## Key APIs and Interfaces + +### Event Types + +```rust +// Core security event +pub enum SecurityEvent { + Syscall(SyscallEvent), + Network(NetworkEvent), + Container(ContainerEvent), + Alert(AlertEvent), +} + +// Syscall event from eBPF +pub struct SyscallEvent { + pub pid: u32, + pub uid: u32, + pub syscall_type: SyscallType, + pub timestamp: DateTime, + pub container_id: Option, +} +``` + +### ML Interface + +```rust +// Feature vector for ML +pub struct SecurityFeatures { + pub syscall_rate: f64, + pub network_rate: f64, + pub unique_processes: u32, + pub privileged_calls: u32, + // ... +} + +// Threat score output +pub enum ThreatScore { + Normal, + Low, + Medium, + High, + Critical, +} +``` + +### Firewall Interface + +```rust +pub trait FirewallBackend { + fn add_rule(&self, rule: &Rule) -> Result<()>; + fn remove_rule(&self, rule: &Rule) -> Result<()>; + fn batch_update(&self, rules: &[Rule]) -> Result<()>; + fn block_container(&self, container_id: &str) -> Result<()>; + fn quarantine_container(&self, container_id: &str) -> Result<()>; +} +``` + +## Configuration + +### Environment Variables + +```bash +APP_HOST=0.0.0.0 +APP_PORT=5000 +DATABASE_URL=stackdog.db +RUST_LOG=info +RUST_BACKTRACE=full + +# Security-specific +EBPF_ENABLED=true +FIREWALL_BACKEND=nftables # or iptables +ML_ENABLED=true +ML_MODEL_PATH=models/ +ALERT_THRESHOLD=0.75 +``` + +### Cargo Features + +```toml +[features] +default = ["nftables", "ml"] +nftables = ["netlink-packet-route"] +iptables = ["iptables"] +ml = ["candle-core", "candle-nn"] +ebpf = ["aya"] +``` + +## Testing Strategy + +### Test Categories + +| Category | Location | Command | Coverage Target | +|----------|----------|---------|-----------------| +| Unit | `src/**/*.rs` | `cargo test` | 80%+ | +| Integration | `tests/integration/` | `cargo test --test integration` | Critical paths | +| E2E | `tests/e2e/` | `cargo test --test e2e` | Key workflows | +| Benchmark | `benches/` | `cargo bench` | Performance targets | + +### Performance Targets + +| Metric | Target | +|--------|--------| +| Event throughput | 100K events/sec | +| ML inference latency | <10ms | +| Firewall update | <1ms per rule | +| Memory usage | <256MB baseline | +| CPU overhead | <5% | + +## Dependencies + +### Core + +- `actix-web` - Web framework +- `aya` - eBPF framework +- `candle-core`, `candle-nn` - ML framework +- `bollard` - Docker API +- `rusqlite` - SQLite driver +- `r2d2` - Connection pool +- `netlink-packet-route` - nftables +- `tokio` - Async runtime + +### Development + +- `mockall` - Mocking for tests +- `criterion` - Benchmarking +- `cargo-audit` - Security audit +- `cargo-deny` - Dependency linting + +## Milestones + +| Version | Target | Features | +|---------|--------|----------| +| v0.1.0 | Week 4 | eBPF collectors, basic rules | +| v0.2.0 | Week 6 | Firewall integration | +| v0.3.0 | Week 10 | ML anomaly detection | +| v0.4.0 | Week 12 | Alerting system | +| v0.5.0 | Week 16 | Web dashboard | +| v1.0.0 | Week 18 | Production release | + +## Risks and Mitigations + +| Risk | Impact | Probability | Mitigation | +|------|--------|-------------|------------| +| eBPF kernel compatibility | High | Medium | Fallback to auditd | +| ML model accuracy | High | Medium | Start with rule-based, iterate | +| Performance overhead | High | Low | Benchmark early, optimize | +| False positives | Medium | High | Tunable thresholds, learning period | + +## Open Questions + +1. **Model Training:** How to collect training data for ML models? + - Decision: Start with synthetic data, then real-world collection + +2. **Multi-node Support:** Single node first, cluster later? + - Decision: Single node for v1.0, cluster in v2.0 + +3. **Kubernetes Support:** Include in scope? + - Decision: Out of scope for v1.0, backlog for v2.0 + +## Resources + +### Documentation + +- [DEVELOPMENT.md](DEVELOPMENT.md) - Full development plan +- [TODO.md](TODO.md) - Task tracking +- [BUGS.md](BUGS.md) - Bug tracking +- [CONTRIBUTING.md](CONTRIBUTING.md) - Contribution guidelines + +### External + +- [Rust Book](https://doc.rust-lang.org/book/) +- [Candle Docs](https://docs.rs/candle-core) +- [aya-rs Docs](https://aya-rs.dev/) +- [eBPF Documentation](https://ebpf.io/) + +## Contact + +- **Project Lead:** Vasili Pascal +- **Email:** info@try.direct +- **Twitter:** [@VasiliiPascal](https://twitter.com/VasiliiPascal) +- **Gitter:** [stackdog/community](https://gitter.im/stackdog/community) + +--- + +*Last updated: 2026-03-13* diff --git a/CHANGELOG.md b/CHANGELOG.md index cd47a13..a6b9bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,71 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +#### Log Sniffing & Analysis (`stackdog sniff`) +- **CLI Subcommands** — Multi-mode binary with `stackdog serve` and `stackdog sniff` + - `--once` flag for single-pass mode + - `--consume` flag to archive logs (zstd) and purge originals + - `--sources` to add custom log paths + - `--ai-provider` to select AI backend (openai/candle) + - `--interval` for polling frequency + - `--output` for archive destination + +- **Log Source Discovery** — Automatic and manual log source management + - System logs (`/var/log/syslog`, `messages`, `auth.log`, etc.) + - Docker container logs via bollard API + - Custom file paths (CLI, env var, or REST API) + - Incremental read position tracking (byte offset persisted in DB) + +- **Log Readers** — Trait-based reader abstraction + - `FileLogReader` with byte-offset tracking and log rotation detection + - `DockerLogReader` using bollard streaming API + - `JournaldReader` (Linux-gated) for systemd journal + +- **AI-Powered Analysis** — Dual-backend log summarization + - `OpenAiAnalyzer` — works with any OpenAI-compatible API (OpenAI, Ollama, vLLM) + - `PatternAnalyzer` — local fallback with error/warning counting and spike detection + - Structured `LogSummary` with anomaly detection (`LogAnomaly`, severity levels) + +- **Log Consumer** — Archive and purge pipeline + - FNV hash-based deduplication + - zstd compression (level 3) for archived logs + - File truncation and Docker log purge + - `ConsumeResult` tracking (entries archived, duplicates skipped, bytes freed) + +- **Reporter** — Bridges log analysis to existing alert system + - Converts `LogAnomaly` → `Alert` using `AlertManager` infrastructure + - Routes notifications via `route_by_severity()` to configured channels + - Persists `LogSummary` records to database + +- **REST API Endpoints** + - `GET /api/logs/sources` — list discovered log sources + - `POST /api/logs/sources` — manually add a custom source + - `GET /api/logs/sources/{path}` — get source details + - `DELETE /api/logs/sources/{path}` — remove a source + - `GET /api/logs/summaries` — list AI-generated summaries (filterable by source) + +- **Database Tables** — `log_sources` and `log_summaries` with indexes + +#### Dependencies +- `clap = "4"` (derive) — CLI argument parsing +- `async-trait = "0.1"` — async trait support +- `reqwest = "0.12"` (json) — HTTP client for AI APIs +- `zstd = "0.13"` — log compression +- `futures-util = "0.3"` — Docker log streaming + +### Changed + +- Refactored `main.rs` to dispatch `serve`/`sniff` subcommands via clap +- Added `events`, `rules`, `alerting`, `models` modules to binary crate +- Updated `.env.sample` with `STACKDOG_LOG_SOURCES`, `STACKDOG_AI_*` config vars + +### Testing + +- **80+ new tests** covering all sniff modules (TDD) + - Config: 12, Discovery: 14, Readers: 10, Analyzer: 16, Consumer: 13, Reporter: 5, Orchestrator: 3, API: 7 + ### Planned - Web dashboard (React/TypeScript) diff --git a/Cargo.toml b/Cargo.toml index fd6a1e1..cf82f97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ tracing-subscriber = "0.3" dotenv = "0.15" anyhow = "1" thiserror = "1" +clap = { version = "4", features = ["derive"] } # Async runtime tokio = { version = "1", features = ["full"] } @@ -37,6 +38,7 @@ actix-web = "4" actix-cors = "0.6" actix-web-actors = "4" actix = "0.13" +async-trait = "0.1" # Database rusqlite = { version = "0.32", features = ["bundled"] } @@ -45,6 +47,15 @@ r2d2 = "0.8" # Docker bollard = "0.16" +# HTTP client (for LLM API) +reqwest = { version = "0.12", features = ["json", "blocking"] } + +# Compression +zstd = "0.13" + +# Stream utilities +futures-util = "0.3" + # eBPF (Linux only) [target.'cfg(target_os = "linux")'.dependencies] aya = "0.12" @@ -66,6 +77,7 @@ ebpf = [] [dev-dependencies] # Testing tokio-test = "0.4" +tempfile = "3" # Benchmarking criterion = { version = "0.5", features = ["html_reports"] } diff --git a/QWEN.md b/QWEN.md new file mode 100644 index 0000000..9ce8ee0 --- /dev/null +++ b/QWEN.md @@ -0,0 +1,311 @@ +# Stackdog Security - Project Context + +## Project Overview + +**Stackdog Security** is a Rust-based security platform for Docker containers and Linux servers. It provides real-time threat detection, AI-powered anomaly detection using Candle (HuggingFace's Rust ML framework), and automated response through firewall management (nftables/iptables). + +### Core Capabilities + +1. **Real-time Monitoring** — System events via eBPF (aya-rs), network traffic, and container activity +2. **AI/ML Detection** — Anomaly detection using Candle (native Rust, no Python) +3. **Automated Response** — Fast nftables/iptables management and container quarantine +4. **Security Dashboard** — Web UI for threat visualization and management + +### Key Technologies + +| Component | Technology | Rationale | +|-----------|-----------|-----------| +| **Core Language** | Rust 2021 | Performance, safety, concurrency | +| **ML Framework** | Candle (HuggingFace) | Native Rust, fast inference, no Python dependencies | +| **eBPF** | aya-rs | Pure Rust eBPF framework, minimal overhead | +| **Firewall** | nftables (netlink) | Modern, faster than iptables | +| **Web Framework** | Actix-web 4.x | High performance | +| **Database** | SQLite + rusqlite + r2d2 | Embedded, low overhead | + +--- + +## Architecture + +``` +stackdog/ +├── src/ +│ ├── collectors/ # Event collection (eBPF, Docker, network) +│ ├── events/ # Event types (SyscallEvent, SecurityEvent) +│ ├── ml/ # ML engine (Candle-based anomaly detection) +│ ├── firewall/ # Firewall management (nftables/iptables) +│ ├── response/ # Automated response actions +│ ├── correlator/ # Event correlation engine +│ ├── alerting/ # Alert system and notifications +│ ├── api/ # REST API + WebSocket +│ ├── config/ # Configuration +│ ├── models/ # Data models +│ ├── database/ # Database operations +│ └── utils/ # Utilities +├── ebpf/ # eBPF programs (separate crate) +├── web/ # React/TypeScript frontend +├── tests/ # Integration and E2E tests +├── benches/ # Performance benchmarks +└── models/ # Pre-trained ML models +``` + +--- + +## Development Status + +**Current Phase:** Phase 1 - Foundation & eBPF Collectors (Weeks 1-4) + +**Active Tasks:** See [TODO.md](TODO.md) + +**Development Plan:** See [DEVELOPMENT.md](DEVELOPMENT.md) + +--- + +## Building and Running + +### Prerequisites + +- Rust 1.75+ (edition 2021) +- SQLite3 + libsqlite3-dev +- Clang + LLVM (for eBPF) +- Kernel 4.19+ (for eBPF with BTF support) +- Docker & Docker Compose (optional) + +### Quick Start + +```bash +# Clone and setup +git clone https://github.com/vsilent/stackdog +cd stackdog + +# Environment setup +cp .env.sample .env + +# Install dependencies (Ubuntu/Debian) +apt-get install libsqlite3-dev libssl-dev clang llvm + +# Build project +cargo build + +# Run tests +cargo test --all + +# Run with debug logging +RUST_LOG=debug cargo run +``` + +### eBPF Development + +```bash +# Install eBPF tools +cargo install cargo-bpf + +# Build eBPF programs +cd ebpf && cargo build --release +``` + +--- + +## Development Commands + +```bash +# Build +cargo build --release + +# Run all tests +cargo test --all + +# Run specific test module +cargo test --test ml::anomaly_detection + +# Linting +cargo clippy --all + +# Formatting +cargo fmt --all -- --check # Check +cargo fmt --all # Fix + +# Performance benchmarks +cargo bench + +# Security audit +cargo audit + +# Watch mode (with cargo-watch) +cargo watch -x test +``` + +--- + +## Testing Strategy (TDD) + +### TDD Workflow + +``` +1. Write failing test +2. Run test (verify failure) +3. Implement minimal code to pass +4. Run test (verify pass) +5. Refactor (maintain passing tests) +``` + +### Test Categories + +| Category | Location | Command | Coverage Target | +|----------|----------|---------|-----------------| +| **Unit Tests** | `src/**/*.rs` | `cargo test` | 80%+ | +| **Integration Tests** | `tests/integration/` | `cargo test --test integration` | Critical paths | +| **E2E Tests** | `tests/e2e/` | `cargo test --test e2e` | Key workflows | +| **Benchmarks** | `benches/` | `cargo bench` | Performance targets | + +### Test Naming Convention + +```rust +#[test] +fn test___() +``` + +Example: +```rust +#[test] +fn test_syscall_event_capture_execve() +#[test] +fn test_isolation_forest_training_valid_data() +#[test] +fn test_container_quarantine_success() +``` + +--- + +## Code Quality Standards + +### Clean Code Principles (Robert C. Martin) + +1. **DRY** - Don't Repeat Yourself +2. **SRP** - Single Responsibility Principle +3. **OCP** - Open/Closed Principle +4. **DIP** - Dependency Inversion Principle +5. **Functional First** - Immutability, `From`/`Into` traits, builder pattern + +### Code Review Checklist + +- [ ] Tests written first (TDD) +- [ ] All tests pass +- [ ] Code formatted (`cargo fmt --all`) +- [ ] No clippy warnings (`cargo clippy --all`) +- [ ] DRY principle followed +- [ ] Functions < 50 lines +- [ ] Error handling comprehensive (`Result` types) +- [ ] Documentation for public APIs + +--- + +## Configuration + +### Environment Variables (`.env`) + +```bash +APP_HOST=0.0.0.0 +APP_PORT=5000 +DATABASE_URL=stackdog.db +RUST_LOG=info +RUST_BACKTRACE=full + +# Security-specific +EBPF_ENABLED=true +FIREWALL_BACKEND=nftables # or iptables +ML_ENABLED=true +ML_MODEL_PATH=models/ +ALERT_THRESHOLD=0.75 +``` + +### Cargo Features + +```toml +[features] +default = ["nftables", "ml"] +nftables = ["netlink-packet-route"] +iptables = ["iptables"] +ml = ["candle-core", "candle-nn"] +ebpf = ["aya"] +``` + +--- + +## Performance Targets + +| Metric | Target | +|--------|--------| +| Event throughput | 100K events/sec | +| ML inference latency | <10ms | +| Firewall update | <1ms per rule | +| Memory usage | <256MB baseline | +| CPU overhead | <5% on monitored host | + +--- + +## Key Files + +| File | Description | +|------|-------------| +| [DEVELOPMENT.md](DEVELOPMENT.md) | Comprehensive development plan with phases | +| [TODO.md](TODO.md) | Task tracking with TDD approach | +| [BUGS.md](BUGS.md) | Bug tracking and reporting | +| [CHANGELOG.md](CHANGELOG.md) | Version history | +| [CONTRIBUTING.md](CONTRIBUTING.md) | Contribution guidelines | +| [ROADMAP.md](ROADMAP.md) | Original roadmap (being updated) | +| `.qwen/PROJECT_MEMORY.md` | Project memory and decisions | + +--- + +## Current Sprint (Phase 1) + +**Goal:** Establish core monitoring infrastructure with eBPF-based syscall collection + +### Active Tasks + +| ID | Task | Status | +|----|------|--------| +| **TASK-001** | Create new project structure for security modules | Pending | +| **TASK-002** | Define security event types | Pending | +| **TASK-003** | Setup aya-rs eBPF integration | Pending | +| **TASK-004** | Implement syscall event capture | Pending | +| **TASK-005** | Create rule engine infrastructure | Pending | + +See [TODO.md](TODO.md) for detailed task descriptions. + +--- + +## Contributing + +1. Pick a task from [TODO.md](TODO.md) or create a new issue +2. Write failing test first (TDD) +3. Implement minimal code to pass +4. Refactor while keeping tests green +5. Submit PR with updated changelog + +### PR Requirements + +- [ ] All tests pass (`cargo test --all`) +- [ ] Code formatted (`cargo fmt --all`) +- [ ] No clippy warnings (`cargo clippy --all`) +- [ ] Changelog updated +- [ ] TDD approach followed + +--- + +## License + +[MIT](LICENSE) + +--- + +## Contact + +- **Project Lead:** Vasili Pascal +- **Email:** info@try.direct +- **Twitter:** [@VasiliiPascal](https://twitter.com/VasiliiPascal) +- **Gitter:** [stackdog/community](https://gitter.im/stackdog/community) + +--- + +*Last updated: 2026-03-13* diff --git a/README.md b/README.md index 41ccfe7..b2fd334 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,13 @@ ### 🔥 Key Features - **📊 Real-time Monitoring** — eBPF-based syscall monitoring with minimal overhead (<5% CPU) -- **🤖 AI/ML Detection** — Candle-powered anomaly detection (native Rust, no Python) +- **🔍 Log Sniffing** — Discover, read, and AI-summarize logs from containers and system files +- **🤖 AI/ML Detection** — Candle-powered anomaly detection + OpenAI/Ollama log analysis - **🚨 Alert System** — Multi-channel notifications (Slack, email, webhook) - **🔒 Automated Response** — nftables/iptables firewall, container quarantine - **📈 Threat Scoring** — Configurable scoring with time-decay - **🎯 Signature Detection** — 10+ built-in threat signatures +- **📦 Log Archival** — Deduplicate and compress logs with zstd, optionally purge originals --- @@ -42,6 +44,17 @@ ## 🚀 Quick Start +### Install with curl (Linux) + +```bash +curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash +``` + +Pin a specific version: +```bash +curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash -s -- --version v0.2.0 +``` + ### Run as Binary ```bash @@ -49,8 +62,30 @@ git clone https://github.com/vsilent/stackdog cd stackdog -# Build and run +# Start the HTTP server (default) cargo run + +# Or explicitly +cargo run -- serve +``` + +### Log Sniffing + +```bash +# Discover and analyze logs (one-shot) +cargo run -- sniff --once + +# Continuous monitoring with AI analysis +cargo run -- sniff --ai-provider openai + +# Use Ollama (local LLM) +STACKDOG_AI_API_URL=http://localhost:11434/v1 cargo run -- sniff + +# Consume mode: archive to zstd + purge originals +cargo run -- sniff --consume --output ./log-archive + +# Add custom log sources +cargo run -- sniff --sources "/var/log/myapp.log,/opt/service/logs" ``` ### Use as Library @@ -106,6 +141,12 @@ docker-compose logs -f stackdog │ │ • Docker │ │ Detection │ │ • Auto-response │ │ │ │ Events │ │ • Scoring │ │ • Alerting │ │ │ └─────────────┘ └─────────────┘ └─────────────────────────┘ │ +│ ┌──────────────────────────────────────────────────────────────┐│ +│ │ Log Sniffing ││ +│ │ • Auto-discovery (system logs, Docker, custom paths) ││ +│ │ • AI summarization (OpenAI/Ollama/Candle) ││ +│ │ • zstd compression, dedup, log purge ││ +│ └──────────────────────────────────────────────────────────────┘│ └─────────────────────────────────────────────────────────────────┘ ``` @@ -118,7 +159,8 @@ docker-compose logs -f stackdog | **Alerting** | Alert management & notifications | ✅ Complete | | **Firewall** | nftables/iptables integration | ✅ Complete | | **Collectors** | eBPF syscall monitoring | ✅ Infrastructure | -| **ML** | Candle-based anomaly detection | 🚧 In progress | +| **Log Sniffing** | Log discovery, AI analysis, archival | ✅ Complete | +| **ML** | Candle-based anomaly detection | ⏳ Planned | --- @@ -251,6 +293,44 @@ let action = ResponseAction::new( - Send alerts - Custom commands +### 7. Log Sniffing & AI Analysis + +```bash +# Discover all log sources and analyze with AI +stackdog sniff --once --ai-provider openai + +# Continuous daemon with local Ollama +stackdog sniff --interval 60 --ai-provider openai + +# Consume: archive (zstd) + purge originals to free disk +stackdog sniff --consume --output ./archive + +# Add custom sources alongside auto-discovered ones +stackdog sniff --sources "/app/logs/api.log,/app/logs/worker.log" +``` + +**Capabilities:** +- 🔍 Auto-discovers system logs, Docker container logs, and custom paths +- 🤖 AI summarization via OpenAI, Ollama, or local pattern analysis +- 📦 Deduplicates and compresses logs with zstd +- 🗑️ Optional `--consume` mode: archives then purges originals +- 📊 Incremental reading — tracks byte offsets, never re-reads old entries +- 🚨 Anomaly alerts routed to configured notification channels + +**REST API:** +```bash +# List discovered sources +curl http://localhost:5000/api/logs/sources + +# Add a custom source +curl -X POST http://localhost:5000/api/logs/sources \ + -H 'Content-Type: application/json' \ + -d '{"path": "/var/log/myapp.log", "name": "My App"}' + +# View AI summaries +curl http://localhost:5000/api/logs/summaries?source_id=myapp +``` + --- ## 📦 Installation @@ -297,6 +377,7 @@ cargo test --lib cargo test --lib -- events:: cargo test --lib -- rules:: cargo test --lib -- alerting:: +cargo test --lib -- sniff:: ``` --- @@ -401,14 +482,377 @@ cargo doc --open ### Project Structure +## 🚀 Quick Start + +### Run as Binary + +```bash +# Clone repository +git clone https://github.com/vsilent/stackdog +cd stackdog + +# Build and run +cargo run +``` + +### Use as Library + +Add to your `Cargo.toml`: + +```toml +[dependencies] +stackdog = "0.2" +``` + +Basic usage: + +```rust +use stackdog::{RuleEngine, AlertManager, ThreatScorer}; + +let mut engine = RuleEngine::new(); +let mut alerts = AlertManager::new()?; +let scorer = ThreatScorer::new(); + +// Process security events +for event in events { + let score = scorer.calculate_score(&event); + if score.is_high_or_higher() { + alerts.generate_alert(...)?; + } +} +``` + +### Docker Development + +```bash +# Start development environment +docker-compose up -d + +# View logs +docker-compose logs -f stackdog +``` + +--- + +## 🏗️ Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Stackdog Security Core │ +├─────────────────────────────────────────────────────────────────┤ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │ +│ │ Collectors │ │ ML/AI │ │ Response Engine │ │ +│ │ │ │ Engine │ │ │ │ +│ │ • eBPF │ │ │ │ • nftables/iptables │ │ +│ │ • Auditd │ │ • Anomaly │ │ • Container quarantine │ │ +│ │ • Docker │ │ Detection │ │ • Auto-response │ │ +│ │ Events │ │ • Scoring │ │ • Alerting │ │ +│ └─────────────┘ └─────────────┘ └─────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Components + +| Component | Description | Status | +|-----------|-------------|--------| +| **Events** | Security event types & validation | ✅ Complete | +| **Rules** | Rule engine & signature detection | ✅ Complete | +| **Alerting** | Alert management & notifications | ✅ Complete | +| **Firewall** | nftables/iptables integration | ✅ Complete | +| **Collectors** | eBPF syscall monitoring | ✅ Infrastructure | +| **ML** | Candle-based anomaly detection | 🚧 In progress | + +--- + +## 🎯 Features + +### 1. Event Collection + +```rust +use stackdog::{SyscallEvent, SyscallType}; + +let event = SyscallEvent::builder() + .pid(1234) + .uid(1000) + .syscall_type(SyscallType::Execve) + .container_id(Some("abc123".to_string())) + .build(); +``` + +**Supported Events:** +- Syscall events (execve, connect, openat, ptrace, etc.) +- Network events +- Container lifecycle events +- Alert events + +### 2. Rule Engine + +```rust +use stackdog::RuleEngine; +use stackdog::rules::builtin::{SyscallBlocklistRule, ProcessExecutionRule}; + +let mut engine = RuleEngine::new(); +engine.register_rule(Box::new(SyscallBlocklistRule::new( + vec![SyscallType::Ptrace, SyscallType::Setuid] +))); + +let results = engine.evaluate(&event); +``` + +**Built-in Rules:** +- Syscall allowlist/blocklist +- Process execution monitoring +- Network connection tracking +- File access monitoring + +### 3. Signature Detection + +```rust +use stackdog::SignatureDatabase; + +let db = SignatureDatabase::new(); +println!("Loaded {} signatures", db.signature_count()); + +let matches = db.detect(&event); +for sig in matches { + println!("Threat: {} (Severity: {})", sig.name(), sig.severity()); +} +``` + +**Built-in Signatures (10+):** +- 🪙 Crypto miner detection +- 🏃 Container escape attempts +- 🌐 Network scanners +- 🔐 Privilege escalation +- 📤 Data exfiltration + +### 4. Threat Scoring + +```rust +use stackdog::ThreatScorer; + +let scorer = ThreatScorer::new(); +let score = scorer.calculate_score(&event); + +if score.is_critical() { + println!("Critical threat detected! Score: {}", score.value()); +} +``` + +**Severity Levels:** +- Info (0-19) +- Low (20-39) +- Medium (40-69) +- High (70-89) +- Critical (90-100) + +### 5. Alert System + +```rust +use stackdog::AlertManager; + +let mut manager = AlertManager::new()?; + +let alert = manager.generate_alert( + AlertType::ThreatDetected, + AlertSeverity::High, + "Suspicious activity detected".to_string(), + Some(event), +)?; + +manager.acknowledge_alert(&alert.id())?; +``` + +**Notification Channels:** +- Console (logging) +- Slack webhooks +- Email (SMTP) +- Generic webhooks + +### 6. Firewall & Response + +```rust +use stackdog::{QuarantineManager, ResponseAction, ResponseType}; + +// Quarantine container +let mut quarantine = QuarantineManager::new()?; +quarantine.quarantine("container_abc123")?; + +// Automated response +let action = ResponseAction::new( + ResponseType::BlockIP("192.168.1.100".to_string()), + "Block malicious IP".to_string(), +); +``` + +**Response Actions:** +- Block IP addresses +- Block ports +- Quarantine containers +- Kill processes +- Send alerts +- Custom commands + +--- + +## 📦 Installation + +### Prerequisites + +- **Rust** 1.75+ ([install](https://rustup.rs/)) +- **SQLite3** + libsqlite3-dev +- **Linux** kernel 4.19+ (for eBPF features) +- **Clang/LLVM** (for eBPF compilation) + +### Install Dependencies + +**Ubuntu/Debian:** +```bash +apt-get install libsqlite3-dev libssl-dev clang llvm pkg-config +``` + +**macOS:** +```bash +brew install sqlite openssl llvm +``` + +**Fedora/RHEL:** +```bash +dnf install sqlite-devel openssl-devel clang llvm +``` + +### Build from Source + +```bash +git clone https://github.com/vsilent/stackdog +cd stackdog +cargo build --release +``` + +### Run Tests + +```bash +# Run all tests +cargo test --lib + +# Run specific module tests +cargo test --lib -- events:: +cargo test --lib -- rules:: +cargo test --lib -- alerting:: +``` + +--- + +## 💡 Usage Examples + +### Example 1: Detect Suspicious Syscalls + +```rust +use stackdog::{RuleEngine, SyscallEvent, SyscallType}; +use stackdog::rules::builtin::SyscallBlocklistRule; + +let mut engine = RuleEngine::new(); +engine.register_rule(Box::new(SyscallBlocklistRule::new( + vec![SyscallType::Ptrace, SyscallType::Setuid] +))); + +let event = SyscallEvent::new( + 1234, 1000, SyscallType::Ptrace, Utc::now() +); + +let results = engine.evaluate(&event); +if results.iter().any(|r| r.is_match()) { + println!("⚠️ Suspicious syscall detected!"); +} +``` + +### Example 2: Container Quarantine + +```rust +use stackdog::QuarantineManager; + +let mut quarantine = QuarantineManager::new()?; + +// Quarantine compromised container +quarantine.quarantine("container_abc123")?; + +// Check quarantine status +let state = quarantine.get_state("container_abc123"); +println!("Container state: {:?}", state); + +// Release after investigation +quarantine.release("container_abc123")?; +``` + +### Example 3: Multi-Event Pattern Detection + +```rust +use stackdog::{SignatureMatcher, PatternMatch, SyscallType}; + +let mut matcher = SignatureMatcher::new(); + +// Detect: execve followed by ptrace (suspicious) +matcher.add_pattern( + PatternMatch::new() + .with_syscall(SyscallType::Execve) + .then_syscall(SyscallType::Ptrace) + .within_seconds(60) +); + +let result = matcher.match_sequence(&events); +if result.is_match() { + println!("⚠️ Suspicious pattern detected!"); +} +``` + +### More Examples + +See [`examples/usage_examples.rs`](examples/usage_examples.rs) for complete working examples. + +Run examples: +```bash +cargo run --example usage_examples +``` + +--- + +## 📚 Documentation + +| Document | Description | +|----------|-------------| +| [DEVELOPMENT.md](DEVELOPMENT.md) | Complete development plan (18 weeks) | +| [TESTING.md](TESTING.md) | Testing guide and infrastructure | +| [TODO.md](TODO.md) | Task tracking and roadmap | +| [CHANGELOG.md](CHANGELOG.md) | Version history | +| [CONTRIBUTING.md](CONTRIBUTING.md) | Contribution guidelines | +| [STATUS.md](STATUS.md) | Current implementation status | + +### API Documentation + +```bash +# Generate docs +cargo doc --open + +# View online (after release) +# https://docs.rs/stackdog ``` stackdog/ ├── src/ +│ ├── cli.rs # Clap CLI (serve/sniff subcommands) │ ├── events/ # Event types & validation │ ├── rules/ # Rule engine & signatures │ ├── alerting/ # Alerts & notifications │ ├── firewall/ # nftables/iptables │ ├── collectors/ # eBPF collectors +│ ├── sniff/ # Log sniffing & AI analysis +│ │ ├── config.rs # SniffConfig (env + CLI) +│ │ ├── discovery.rs # Log source auto-discovery +│ │ ├── reader.rs # File/Docker/Journald readers +│ │ ├── analyzer.rs # AI summarization (OpenAI + pattern) +│ │ ├── consumer.rs # zstd compression, dedup, purge +│ │ └── reporter.rs # Alert routing +│ ├── api/ # REST API endpoints +│ ├── database/ # SQLite + repositories │ ├── ml/ # ML infrastructure │ └── config/ # Configuration ├── examples/ # Usage examples @@ -507,11 +951,12 @@ Look for issues labeled: - ✅ Signature detection (TASK-006) - ✅ Alert system (TASK-007) - ✅ Firewall integration (TASK-008) +- ✅ Log sniffing & AI analysis (TASK-009) ### Upcoming Tasks -- ⏳ Web dashboard (TASK-009) - ⏳ ML anomaly detection (TASK-010) +- ⏳ Web dashboard (TASK-011) - ⏳ Kubernetes support (BACKLOG) --- diff --git a/docker-compose.yml b/docker-compose.yml index 381f647..289a2fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,20 +16,15 @@ services: - | echo "Waiting for dependencies..." sleep 5 - echo "Running migrations..." - diesel migration run || echo "Migrations may have already run" echo "Starting Stackdog..." cargo run --bin stackdog ports: - - "5003:5000" + - "${APP_PORT:-8080}:${APP_PORT:-8080}" env_file: - .env environment: - RUST_LOG=debug - RUST_BACKTRACE=full - - APP_HOST=0.0.0.0 - - APP_PORT=5000 - - DATABASE_URL=/app/db/stackdog.db volumes: - db_data:/app/db - ./.env:/app/.env:ro diff --git a/docs/DAY1_PROGRESS.md b/docs/DAY1_PROGRESS.md new file mode 100644 index 0000000..7f5e93a --- /dev/null +++ b/docs/DAY1_PROGRESS.md @@ -0,0 +1,124 @@ +# Day 1 Progress Report - Database Integration + +**Date:** 2026-03-16 +**Status:** ⚠️ Partial Progress + +--- + +## What Was Accomplished + +### ✅ Database Schema Created +- 3 migration files created +- Alerts, threats, containers_cache tables defined +- Indexes for performance + +### ✅ Database Layer Structure +- `src/database/connection.rs` - Connection pool +- `src/database/models/` - Data models +- `src/database/repositories/` - Repository pattern + +### ✅ API Integration Started +- Alerts API updated to use database +- Dependency injection configured +- Main.rs updated with database initialization + +--- + +## Current Blockers + +### Diesel Version Compatibility +The current diesel version (1.4) has API incompatibilities with the migration system. + +**Options:** +1. Upgrade to diesel 2.x (breaking changes) +2. Use raw SQL for everything (more work) +3. Simplify to basic SQL queries (recommended for now) + +--- + +## Recommended Next Steps + +### Option A: Quick Fix (1-2 hours) +Use rusqlite directly instead of diesel: +```toml +[dependencies] +rusqlite = { version = "0.31", features = ["bundled"] } +``` + +Benefits: +- Simpler API +- No migration issues +- Less boilerplate + +### Option B: Full Diesel Upgrade (Half day) +Upgrade to diesel 2.x: +- Update Cargo.toml +- Fix breaking changes +- Update all queries + +### Option C: Hybrid Approach (Recommended) +- Use diesel for connection pooling +- Use raw SQL for queries +- Keep current structure + +--- + +## Files Created Today + +### Migrations +- `migrations/00000000000000_create_alerts/up.sql` +- `migrations/00000000000000_create_alerts/down.sql` +- `migrations/00000000000001_create_threats/*` +- `migrations/00000000000002_create_containers_cache/*` + +### Database Layer +- `src/database/connection.rs` +- `src/database/models/mod.rs` +- `src/database/repositories/alerts.rs` +- `src/database/repositories/mod.rs` +- `src/database/mod.rs` + +### API Updates +- `src/api/alerts.rs` - Updated with DB integration +- `src/main.rs` - Database initialization + +--- + +## Time Spent + +| Task | Time | +|------|------| +| Schema design | 30 min | +| Migration files | 30 min | +| Database layer | 2 hours | +| API integration | 1 hour | +| Debugging diesel | 1 hour | +| **Total** | **5 hours** | + +--- + +## Remaining Work for Day 1 + +### To Complete Database Integration +1. Fix diesel compatibility (30 min) +2. Test database initialization (15 min) +3. Test alert CRUD operations (30 min) +4. Update remaining API endpoints (1 hour) + +**Estimated time:** 2.5 hours + +--- + +## Decision Point + +**Choose one:** + +1. **Continue with diesel** - Fix compatibility issues +2. **Switch to rusqlite** - Simpler, faster implementation +3. **Hybrid approach** - Keep diesel for pooling, raw SQL for queries + +**Recommendation:** Option 3 (Hybrid) - Best balance of speed and maintainability + +--- + +*Report generated: 2026-03-16* diff --git a/docs/DAY2_PLAN.md b/docs/DAY2_PLAN.md new file mode 100644 index 0000000..b9ebeaf --- /dev/null +++ b/docs/DAY2_PLAN.md @@ -0,0 +1,47 @@ +# Day 2: Docker Integration + +**Date:** 2026-03-16 +**Goal:** Connect to Docker API and list real containers + +--- + +## Morning: Docker Client Setup + +### Tasks +- [x] Add bollard dependency +- [ ] Create Docker client wrapper +- [ ] Test Docker connection +- [ ] List containers + +### Files to Create +``` +src/docker/ +├── mod.rs +├── client.rs # Docker client wrapper +├── containers.rs # Container operations +└── types.rs # Type conversions +``` + +--- + +## Afternoon: Container Management + +### Tasks +- [ ] Implement container listing +- [ ] Implement quarantine (disconnect network) +- [ ] Implement release (reconnect network) +- [ ] Cache container data in DB + +--- + +## Success Criteria + +- [ ] Can list real Docker containers +- [ ] Can get container details +- [ ] Quarantine actually disconnects network +- [ ] Release reconnects network +- [ ] All tests passing + +--- + +*Plan created: 2026-03-16* diff --git a/docs/DAY2_PROGRESS.md b/docs/DAY2_PROGRESS.md new file mode 100644 index 0000000..002faef --- /dev/null +++ b/docs/DAY2_PROGRESS.md @@ -0,0 +1,126 @@ +# Day 2 Progress Report - Docker Integration + +**Date:** 2026-03-16 +**Status:** ⚠️ Partial Progress + +--- + +## What Was Accomplished + +### ✅ Docker Module Structure Created +- `src/docker/client.rs` - Docker client wrapper +- `src/docker/containers.rs` - Container management +- `src/docker/mod.rs` - Module exports + +### ✅ Docker Client Implementation +- Connection to Docker daemon +- List containers +- Get container info +- Quarantine (disconnect networks) +- Release (reconnect) + +### ✅ Container Manager +- High-level container operations +- Alert generation on quarantine +- Security status calculation + +### ✅ Containers API +- `GET /api/containers` - List containers +- `POST /api/containers/:id/quarantine` - Quarantine container +- `POST /api/containers/:id/release` - Release container +- Fallback to mock data if Docker unavailable + +--- + +## Current Blockers + +### Bollard Crate Linking +The bollard crate isn't linking properly in the binary. + +**Errors:** +- `can't find crate for bollard` +- Type annotation issues in API handlers + +**Possible Causes:** +1. Bollard needs to be in lib.rs extern crate +2. Version incompatibility +3. Feature flags needed + +--- + +## Files Created (4 files) + +### Docker Module +- `src/docker/client.rs` (176 lines) +- `src/docker/containers.rs` (144 lines) +- `src/docker/mod.rs` (8 lines) + +### API +- `src/api/containers.rs` (updated, 168 lines) + +### Documentation +- `docs/DAY2_PLAN.md` +- `docs/DAY2_PROGRESS.md` + +--- + +## Time Spent + +| Task | Time | +|------|------| +| Docker client implementation | 1.5 hours | +| Container manager | 1 hour | +| Containers API | 1 hour | +| Debugging bollard linking | 1.5 hours | +| **Total** | **5 hours** | + +--- + +## Remaining Work + +### To Complete Docker Integration +1. Fix bollard crate linking (30 min) +2. Test with real Docker daemon (30 min) +3. Add container security scanning (1 hour) +4. Add threat detection rules (1 hour) + +**Estimated time:** 3 hours + +--- + +## Recommended Next Steps + +### Option A: Fix Bollard Linking (Recommended) +Add bollard to lib.rs: +```rust +#[cfg(target_os = "linux")] +extern crate bollard; +``` + +Then fix type annotations in API handlers. + +### Option B: Use Docker CLI Instead +Use `std::process::Command` to run docker commands: +```rust +Command::new("docker").arg("ps").output() +``` + +Simpler but less elegant. + +### Option C: Mock for Now +Keep mock data, implement real Docker later. + +--- + +## Decision Point + +**Choose one:** +1. **Fix bollard** - Continue with current approach (30 min) +2. **Use docker CLI** - Switch to command-line approach +3. **Mock for now** - Focus on other features + +**Recommendation:** Option 1 - Fix bollard linking, it's almost working. + +--- + +*Report generated: 2026-03-16* diff --git a/docs/REAL_FUNCTIONALITY_PLAN.md b/docs/REAL_FUNCTIONALITY_PLAN.md new file mode 100644 index 0000000..a5ebdbd --- /dev/null +++ b/docs/REAL_FUNCTIONALITY_PLAN.md @@ -0,0 +1,410 @@ +# Real Functionality Implementation Plan + +**Goal:** Add real Docker integration and database persistence +**Timeline:** 3-5 days +**Target Release:** v0.3.0 "Alpha" + +--- + +## Day 1: Database Integration + +### Morning: SQLite Schema & Migrations + +**Tasks:** +1. Create database schema +2. Write SQL migrations +3. Test migration execution + +**Files:** +``` +migrations/ +├── 00000000000000_create_alerts/ +│ ├── up.sql +│ └── down.sql +├── 00000000000001_create_threats/ +│ ├── up.sql +│ └── down.sql +└── 00000000000002_create_containers_cache/ + ├── up.sql + └── down.sql +``` + +**Schema:** +```sql +-- Alerts table +CREATE TABLE alerts ( + id TEXT PRIMARY KEY, + alert_type TEXT NOT NULL, + severity TEXT NOT NULL, + message TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'New', + timestamp DATETIME NOT NULL, + metadata TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Threats table +CREATE TABLE threats ( + id TEXT PRIMARY KEY, + threat_type TEXT NOT NULL, + severity TEXT NOT NULL, + score INTEGER NOT NULL, + source TEXT NOT NULL, + timestamp DATETIME NOT NULL, + status TEXT NOT NULL DEFAULT 'New', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Containers cache table +CREATE TABLE containers_cache ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + image TEXT NOT NULL, + status TEXT NOT NULL, + risk_score INTEGER DEFAULT 0, + last_updated DATETIME DEFAULT CURRENT_TIMESTAMP +); +``` + +**Tests:** +- Migration runs successfully +- Tables created correctly +- Can insert/query data + +--- + +### Afternoon: Database Repository Layer + +**Tasks:** +1. Create repository traits +2. Implement AlertRepository +3. Implement ThreatRepository +4. Implement ContainerRepository + +**Files:** +``` +src/database/ +├── mod.rs +├── connection.rs # DB connection pool +├── repositories/ +│ ├── mod.rs +│ ├── alerts.rs +│ ├── threats.rs +│ └── containers.rs +└── models/ + ├── mod.rs + ├── alert.rs + ├── threat.rs + └── container.rs +``` + +**Implementation:** +```rust +// src/database/repositories/alerts.rs +pub trait AlertRepository: Send + Sync { + async fn list(&self, filter: AlertFilter) -> Result>; + async fn get(&self, id: &str) -> Result>; + async fn create(&self, alert: Alert) -> Result; + async fn update_status(&self, id: &str, status: AlertStatus) -> Result<()>; + async fn get_stats(&self) -> Result; +} +``` + +**Tests:** +- Can create alert +- Can list alerts with filter +- Can update status +- Stats calculation correct + +--- + +## Day 2: Docker Integration + +### Morning: Docker Client Setup + +**Tasks:** +1. Add bollard dependency +2. Create Docker client wrapper +3. Test Docker connection +4. List containers + +**Files:** +``` +src/docker/ +├── mod.rs +├── client.rs # Docker client wrapper +├── containers.rs # Container operations +└── types.rs # Docker type conversions +``` + +**Implementation:** +```rust +// src/docker/client.rs +pub struct DockerClient { + client: bollard::Docker, +} + +impl DockerClient { + pub fn new() -> Result; + pub async fn list_containers(&self) -> Result>; + pub async fn get_container(&self, id: &str) -> Result; + pub async fn quarantine_container(&self, id: &str) -> Result<()>; + pub async fn release_container(&self, id: &str) -> Result<()>; +} +``` + +**Tests:** +- Docker client connects +- Can list containers +- Can get container details + +--- + +### Afternoon: Container Management + +**Tasks:** +1. Implement container listing +2. Implement quarantine (disconnect network) +3. Implement release (reconnect network) +4. Cache container data in DB + +**Implementation:** +```rust +// Quarantine implementation +pub async fn quarantine_container(&self, id: &str) -> Result<()> { + // Disconnect from all networks + let networks = self.client.list_networks().await?; + for network in networks { + self.client.disconnect_network( + &network.name, + NetworkDisconnectOptions { + container_id: Some(id.to_string()), + ..Default::default() + } + ).await?; + } + Ok(()) +} +``` + +**Tests:** +- List real containers from Docker +- Quarantine actually disconnects network +- Release reconnects network + +--- + +## Day 3: Connect API to Real Data + +### Morning: Update API Endpoints + +**Tasks:** +1. Inject repositories into API handlers +2. Replace mock data with DB queries +3. Test all endpoints + +**Changes:** +```rust +// Before (mock) +pub async fn get_alerts() -> impl Responder { + let alerts = vec![/* mock data */]; + HttpResponse::Ok().json(alerts) +} + +// After (real) +pub async fn get_alerts( + repo: web::Data, + query: web::Query +) -> impl Responder { + let filter = AlertFilter::from(query); + let alerts = repo.list(filter).await?; + HttpResponse::Ok().json(alerts) +} +``` + +**Endpoints to Update:** +- [ ] `GET /api/alerts` - Query database +- [ ] `GET /api/alerts/stats` - Calculate from DB +- [ ] `POST /api/alerts/:id/acknowledge` - Update DB +- [ ] `POST /api/alerts/:id/resolve` - Update DB +- [ ] `GET /api/containers` - Query Docker + cache +- [ ] `POST /api/containers/:id/quarantine` - Call Docker API +- [ ] `POST /api/containers/:id/release` - Call Docker API +- [ ] `GET /api/threats` - Query database +- [ ] `GET /api/threats/statistics` - Calculate from DB + +--- + +### Afternoon: Testing & Bug Fixes + +**Tasks:** +1. Test each endpoint with real data +2. Fix any bugs +3. Add error handling +4. Performance testing + +**Test Script:** +```bash +# Test alerts endpoint +curl http://localhost:5000/api/alerts + +# Test containers endpoint +curl http://localhost:5000/api/containers + +# Test quarantine +curl -X POST http://localhost:5000/api/containers/test123/quarantine +``` + +--- + +## Day 4: Real-Time Events + +### Morning: Event Generation + +**Tasks:** +1. Create event generator service +2. Generate alerts from Docker events +3. Store events in database + +**Implementation:** +```rust +// Listen to Docker events +pub async fn listen_docker_events( + client: DockerClient, + alert_repo: Arc +) { + let mut events = client.events().await; + while let Some(event) = events.next().await { + match event { + DockerEvent::ContainerStart { id, name } => { + alert_repo.create(Alert::new( + AlertType::SystemEvent, + AlertSeverity::Info, + format!("Container {} started", name) + )).await?; + } + DockerEvent::ContainerDie { id, name } => { + // Check if container was quarantined + } + _ => {} + } + } +} +``` + +--- + +### Afternoon: WebSocket Real-Time Updates + +**Tasks:** +1. Implement proper WebSocket with actix-web-actors +2. Broadcast events to connected clients +3. Test real-time updates + +--- + +## Day 5: Polish & Release Prep + +### Morning: Security Features + +**Tasks:** +1. Add basic threat detection rules +2. Generate alerts from suspicious activity +3. Test detection accuracy + +**Example Rules:** +```rust +// Rule: Container running as root +if container.user == "root" { + generate_alert(AlertSeverity::Medium, "Container running as root"); +} + +// Rule: Container with privileged mode +if container.privileged { + generate_alert(AlertSeverity::High, "Container in privileged mode"); +} +``` + +--- + +### Afternoon: Release Preparation + +**Tasks:** +1. Update CHANGELOG.md +2. Update README.md with real features +3. Write release notes +4. Create git tag v0.3.0-alpha +5. Test release build + +--- + +## Success Criteria + +### Must Have (for v0.3.0-alpha) + +- [ ] Alerts stored in SQLite +- [ ] Can list real Docker containers +- [ ] Can actually quarantine container +- [ ] Can actually release container +- [ ] Alert acknowledge/resolve persists +- [ ] All API endpoints use real data + +### Nice to Have + +- [ ] Real-time WebSocket updates +- [ ] Docker event listening +- [ ] Basic threat detection rules +- [ ] Container risk scoring + +### Future (v0.4.0+) + +- [ ] eBPF syscall monitoring +- [ ] ML anomaly detection +- [ ] Advanced threat detection +- [ ] Network traffic analysis + +--- + +## Risks & Mitigations + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Docker API changes | Medium | Use stable bollard version | +| SQLite concurrency | Low | Use connection pool | +| WebSocket complexity | Medium | Use polling as fallback | +| Performance issues | Medium | Add caching layer | + +--- + +## Testing Checklist + +### Database +- [ ] Migrations run successfully +- [ ] Can insert alerts +- [ ] Can query alerts with filters +- [ ] Can update alert status +- [ ] Stats calculation correct + +### Docker +- [ ] Can list containers +- [ ] Can get container details +- [ ] Quarantine disconnects network +- [ ] Release reconnects network +- [ ] Works with running containers + +### API +- [ ] All endpoints return real data +- [ ] Error handling works +- [ ] CORS works +- [ ] Performance acceptable + +### Frontend +- [ ] Dashboard shows real containers +- [ ] Can acknowledge alerts +- [ ] Can resolve alerts +- [ ] Quarantine button works +- [ ] Release button works + +--- + +*Plan created: 2026-03-15* diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..11e9942 --- /dev/null +++ b/install.sh @@ -0,0 +1,148 @@ +#!/bin/sh +# Stackdog Security — install script +# +# Usage: +# curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash +# curl -fsSL https://raw.githubusercontent.com/vsilent/stackdog/dev/install.sh | sudo bash -s -- --version v0.2.0 +# +# Installs the stackdog binary to /usr/local/bin. +# Requires: curl, tar, sha256sum (or shasum), Linux x86_64 or aarch64. + +set -eu + +REPO="vsilent/stackdog" +INSTALL_DIR="/usr/local/bin" +BINARY_NAME="stackdog" + +# --- helpers ---------------------------------------------------------------- + +info() { printf '\033[1;32m▸ %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33m⚠ %s\033[0m\n' "$*"; } +error() { printf '\033[1;31m✖ %s\033[0m\n' "$*" >&2; exit 1; } + +need_cmd() { + if ! command -v "$1" > /dev/null 2>&1; then + error "Required command not found: $1" + fi +} + +# --- detect platform -------------------------------------------------------- + +detect_platform() { + OS="$(uname -s)" + ARCH="$(uname -m)" + + case "$OS" in + Linux) OS="linux" ;; + *) error "Unsupported OS: $OS. Stackdog binaries are available for Linux only." ;; + esac + + case "$ARCH" in + x86_64|amd64) ARCH="x86_64" ;; + aarch64|arm64) ARCH="aarch64" ;; + *) error "Unsupported architecture: $ARCH. Supported: x86_64, aarch64." ;; + esac + + PLATFORM="${OS}-${ARCH}" +} + +# --- resolve version -------------------------------------------------------- + +resolve_version() { + if [ -n "${VERSION:-}" ]; then + # strip leading v if present for consistency + VERSION="$(echo "$VERSION" | sed 's/^v//')" + TAG="v${VERSION}" + return + fi + + info "Fetching latest release..." + TAG="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" \ + | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/')" + + if [ -z "$TAG" ]; then + error "Could not determine latest release. Specify a version with --version" + fi + + VERSION="$(echo "$TAG" | sed 's/^v//')" +} + +# --- download & verify ------------------------------------------------------ + +download_and_install() { + TARBALL="${BINARY_NAME}-${PLATFORM}.tar.gz" + CHECKSUM_FILE="${TARBALL}.sha256" + DOWNLOAD_URL="https://github.com/${REPO}/releases/download/${TAG}/${TARBALL}" + CHECKSUM_URL="https://github.com/${REPO}/releases/download/${TAG}/${CHECKSUM_FILE}" + + TMPDIR="$(mktemp -d)" + trap 'rm -rf "$TMPDIR"' EXIT + + info "Downloading stackdog ${VERSION} for ${PLATFORM}..." + curl -fsSL -o "${TMPDIR}/${TARBALL}" "$DOWNLOAD_URL" \ + || error "Download failed. Check that release ${TAG} exists at https://github.com/${REPO}/releases" + + info "Downloading checksum..." + curl -fsSL -o "${TMPDIR}/${CHECKSUM_FILE}" "$CHECKSUM_URL" \ + || warn "Checksum file not available — skipping verification" + + # verify checksum if available + if [ -f "${TMPDIR}/${CHECKSUM_FILE}" ]; then + info "Verifying checksum..." + EXPECTED="$(awk '{print $1}' "${TMPDIR}/${CHECKSUM_FILE}")" + if command -v sha256sum > /dev/null 2>&1; then + ACTUAL="$(sha256sum "${TMPDIR}/${TARBALL}" | awk '{print $1}')" + elif command -v shasum > /dev/null 2>&1; then + ACTUAL="$(shasum -a 256 "${TMPDIR}/${TARBALL}" | awk '{print $1}')" + else + warn "sha256sum/shasum not found — skipping checksum verification" + ACTUAL="$EXPECTED" + fi + + if [ "$EXPECTED" != "$ACTUAL" ]; then + error "Checksum mismatch!\n expected: ${EXPECTED}\n actual: ${ACTUAL}" + fi + fi + + info "Extracting..." + tar -xzf "${TMPDIR}/${TARBALL}" -C "${TMPDIR}" + + info "Installing to ${INSTALL_DIR}/${BINARY_NAME}..." + install -m 755 "${TMPDIR}/${BINARY_NAME}" "${INSTALL_DIR}/${BINARY_NAME}" +} + +# --- main ------------------------------------------------------------------- + +main() { + # parse args + while [ $# -gt 0 ]; do + case "$1" in + --version) VERSION="$2"; shift 2 ;; + --help|-h) + echo "Usage: install.sh [--version VERSION]" + echo "" + echo "Install stackdog binary to ${INSTALL_DIR}." + echo "" + echo "Options:" + echo " --version VERSION Install a specific version (e.g. v0.2.0)" + echo " --help Show this help" + exit 0 + ;; + *) error "Unknown option: $1" ;; + esac + done + + need_cmd curl + need_cmd tar + + detect_platform + resolve_version + download_and_install + + info "stackdog ${VERSION} installed successfully!" + echo "" + echo " Run: stackdog --help" + echo "" +} + +main "$@" diff --git a/src/alerting/notifications.rs b/src/alerting/notifications.rs index aa5f25a..d35d7e0 100644 --- a/src/alerting/notifications.rs +++ b/src/alerting/notifications.rs @@ -111,14 +111,39 @@ impl NotificationChannel { Ok(NotificationResult::Success("sent to console".to_string())) } - /// Send to Slack + /// Send to Slack via incoming webhook fn send_slack(&self, alert: &Alert, config: &NotificationConfig) -> Result { - // In production, this would make HTTP request to Slack webhook - // For now, just log - if config.slack_webhook().is_some() { - log::info!("Would send to Slack: {}", alert.message()); - Ok(NotificationResult::Success("sent to Slack".to_string())) + if let Some(webhook_url) = config.slack_webhook() { + let payload = build_slack_message(alert); + log::debug!("Sending Slack notification to webhook"); + log::trace!("Slack payload: {}", payload); + + // Blocking HTTP POST — notification sending is synchronous in this codebase + let client = reqwest::blocking::Client::new(); + match client + .post(webhook_url) + .header("Content-Type", "application/json") + .body(payload) + .send() + { + Ok(resp) => { + if resp.status().is_success() { + log::info!("Slack notification sent successfully"); + Ok(NotificationResult::Success("sent to Slack".to_string())) + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + log::warn!("Slack API returned {}: {}", status, body); + Ok(NotificationResult::Failure(format!("Slack returned {}: {}", status, body))) + } + } + Err(e) => { + log::warn!("Failed to send Slack notification: {}", e); + Ok(NotificationResult::Failure(format!("Slack request failed: {}", e))) + } + } } else { + log::debug!("Slack webhook not configured, skipping"); Ok(NotificationResult::Failure("Slack webhook not configured".to_string())) } } @@ -211,27 +236,19 @@ pub fn severity_to_slack_color(severity: AlertSeverity) -> &'static str { /// Build Slack message payload pub fn build_slack_message(alert: &Alert) -> String { - format!( - r#"{{ - "text": "Security Alert", - "attachments": [{{ - "color": "{}", - "title": "{:?} ", - "text": "{}", - "fields": [ - {{"title": "Severity", "value": "{}", "short": true}}, - {{"title": "Status", "value": "{}", "short": true}}, - {{"title": "Time", "value": "{}", "short": true}} - ] - }}] - }}"#, - severity_to_slack_color(alert.severity()), - alert.alert_type(), - alert.message(), - alert.severity(), - alert.status(), - alert.timestamp() - ) + serde_json::json!({ + "text": "🐕 Stackdog Security Alert", + "attachments": [{ + "color": severity_to_slack_color(alert.severity()), + "title": format!("{:?}", alert.alert_type()), + "text": alert.message(), + "fields": [ + {"title": "Severity", "value": alert.severity().to_string(), "short": true}, + {"title": "Status", "value": alert.status().to_string(), "short": true}, + {"title": "Time", "value": alert.timestamp().to_rfc3339(), "short": true} + ] + }] + }).to_string() } /// Build webhook payload diff --git a/src/api/logs.rs b/src/api/logs.rs new file mode 100644 index 0000000..9963c33 --- /dev/null +++ b/src/api/logs.rs @@ -0,0 +1,277 @@ +//! Log sources and summaries API endpoints + +use actix_web::{web, HttpResponse, Responder}; +use serde::Deserialize; +use crate::database::connection::DbPool; +use crate::database::repositories::log_sources; +use crate::sniff::discovery::{LogSource, LogSourceType}; + +/// Query parameters for summary filtering +#[derive(Debug, Deserialize)] +pub struct SummaryQuery { + source_id: Option, +} + +/// Request body for adding a custom log source +#[derive(Debug, Deserialize)] +pub struct AddSourceRequest { + pub path: String, + pub name: Option, +} + +/// List all discovered log sources +/// +/// GET /api/logs/sources +pub async fn list_sources(pool: web::Data) -> impl Responder { + match log_sources::list_log_sources(&pool) { + Ok(sources) => HttpResponse::Ok().json(sources), + Err(e) => { + log::error!("Failed to list log sources: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to list log sources" + })) + } + } +} + +/// Get a single log source by path +/// +/// GET /api/logs/sources/{path} +pub async fn get_source(pool: web::Data, path: web::Path) -> impl Responder { + match log_sources::get_log_source_by_path(&pool, &path) { + Ok(Some(source)) => HttpResponse::Ok().json(source), + Ok(None) => HttpResponse::NotFound().json(serde_json::json!({ + "error": "Log source not found" + })), + Err(e) => { + log::error!("Failed to get log source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to get log source" + })) + } + } +} + +/// Manually add a custom log source +/// +/// POST /api/logs/sources +pub async fn add_source( + pool: web::Data, + body: web::Json, +) -> impl Responder { + let name = body.name.clone().unwrap_or_else(|| body.path.clone()); + let source = LogSource::new(LogSourceType::CustomFile, body.path.clone(), name); + + match log_sources::upsert_log_source(&pool, &source) { + Ok(_) => HttpResponse::Created().json(source), + Err(e) => { + log::error!("Failed to add log source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to add log source" + })) + } + } +} + +/// Delete a log source +/// +/// DELETE /api/logs/sources/{path} +pub async fn delete_source(pool: web::Data, path: web::Path) -> impl Responder { + match log_sources::delete_log_source(&pool, &path) { + Ok(_) => HttpResponse::NoContent().finish(), + Err(e) => { + log::error!("Failed to delete log source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to delete log source" + })) + } + } +} + +/// List AI-generated log summaries +/// +/// GET /api/logs/summaries +pub async fn list_summaries( + pool: web::Data, + query: web::Query, +) -> impl Responder { + let source_id = query.source_id.as_deref().unwrap_or(""); + if source_id.is_empty() { + // List all summaries — check each known source + match log_sources::list_log_sources(&pool) { + Ok(sources) => { + let mut all_summaries = Vec::new(); + for source in &sources { + if let Ok(summaries) = log_sources::list_summaries_for_source(&pool, &source.path_or_id) { + all_summaries.extend(summaries); + } + } + HttpResponse::Ok().json(all_summaries) + } + Err(e) => { + log::error!("Failed to list summaries: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to list summaries" + })) + } + } + } else { + match log_sources::list_summaries_for_source(&pool, source_id) { + Ok(summaries) => HttpResponse::Ok().json(summaries), + Err(e) => { + log::error!("Failed to list summaries for source: {}", e); + HttpResponse::InternalServerError().json(serde_json::json!({ + "error": "Failed to list summaries" + })) + } + } + } +} + +/// Configure log API routes +pub fn configure_routes(cfg: &mut web::ServiceConfig) { + cfg.service( + web::scope("/api/logs") + .route("/sources", web::get().to(list_sources)) + .route("/sources", web::post().to(add_source)) + .route("/sources/{path}", web::get().to(get_source)) + .route("/sources/{path}", web::delete().to(delete_source)) + .route("/summaries", web::get().to(list_summaries)) + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use actix_web::{test, App}; + use crate::database::connection::{create_pool, init_database}; + + fn setup_pool() -> DbPool { + let pool = create_pool(":memory:").unwrap(); + init_database(&pool).unwrap(); + pool + } + + #[actix_rt::test] + async fn test_list_sources_empty() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get().uri("/api/logs/sources").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + } + + #[actix_rt::test] + async fn test_add_source() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let body = serde_json::json!({ "path": "/var/log/test.log", "name": "Test Log" }); + let req = test::TestRequest::post() + .uri("/api/logs/sources") + .set_json(&body) + .to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 201); + } + + #[actix_rt::test] + async fn test_add_and_list_sources() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + // Add a source + let body = serde_json::json!({ "path": "/var/log/app.log" }); + let req = test::TestRequest::post() + .uri("/api/logs/sources") + .set_json(&body) + .to_request(); + test::call_service(&app, req).await; + + // List sources + let req = test::TestRequest::get().uri("/api/logs/sources").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + + let body: Vec = test::read_body_json(resp).await; + assert_eq!(body.len(), 1); + } + + #[actix_rt::test] + async fn test_get_source_not_found() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get().uri("/api/logs/sources/nonexistent").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 404); + } + + #[actix_rt::test] + async fn test_delete_source() { + let pool = setup_pool(); + + // Add source directly via repository (avoids route path issues) + let source = LogSource::new(LogSourceType::CustomFile, "test-delete.log".into(), "Test Delete".into()); + log_sources::upsert_log_source(&pool, &source).unwrap(); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::delete() + .uri("/api/logs/sources/test-delete.log") + .to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 204); + } + + #[actix_rt::test] + async fn test_list_summaries_empty() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get().uri("/api/logs/summaries").to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + } + + #[actix_rt::test] + async fn test_list_summaries_filtered() { + let pool = setup_pool(); + let app = test::init_service( + App::new() + .app_data(web::Data::new(pool)) + .configure(configure_routes) + ).await; + + let req = test::TestRequest::get() + .uri("/api/logs/summaries?source_id=test-source") + .to_request(); + let resp = test::call_service(&app, req).await; + assert_eq!(resp.status(), 200); + } +} diff --git a/src/api/mod.rs b/src/api/mod.rs index 754a6d5..6120aab 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,6 +7,7 @@ pub mod alerts; pub mod containers; pub mod threats; pub mod websocket; +pub mod logs; /// Marker struct for module tests pub struct ApiMarker; @@ -17,6 +18,7 @@ pub use alerts::configure_routes as configure_alerts_routes; pub use containers::configure_routes as configure_containers_routes; pub use threats::configure_routes as configure_threats_routes; pub use websocket::configure_routes as configure_websocket_routes; +pub use logs::configure_routes as configure_logs_routes; /// Configure all API routes pub fn configure_all_routes(cfg: &mut actix_web::web::ServiceConfig) { @@ -25,4 +27,5 @@ pub fn configure_all_routes(cfg: &mut actix_web::web::ServiceConfig) { configure_containers_routes(cfg); configure_threats_routes(cfg); configure_websocket_routes(cfg); + configure_logs_routes(cfg); } diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..ea26fcc --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,173 @@ +//! CLI argument parsing for Stackdog +//! +//! Defines the command-line interface using clap derive macros. +//! Supports `serve` (HTTP server) and `sniff` (log analysis) subcommands. + +use clap::{Parser, Subcommand}; + +/// Stackdog Security — Docker & Linux server security platform +#[derive(Parser, Debug)] +#[command(name = "stackdog", version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Option, +} + +/// Available subcommands +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Start the HTTP API server (default behavior) + Serve, + + /// Sniff and analyze logs from Docker containers and system sources + Sniff { + /// Run a single scan/analysis pass, then exit + #[arg(long)] + once: bool, + + /// Consume logs: archive to zstd, then purge originals to free disk + #[arg(long)] + consume: bool, + + /// Output directory for consumed logs + #[arg(long, default_value = "./stackdog-logs/")] + output: String, + + /// Additional log file paths to watch (comma-separated) + #[arg(long)] + sources: Option, + + /// Poll interval in seconds + #[arg(long, default_value = "30")] + interval: u64, + + /// AI provider: "openai", "ollama", or "candle" + #[arg(long)] + ai_provider: Option, + + /// AI model name (e.g. "gpt-4o-mini", "qwen2.5-coder:latest", "llama3") + #[arg(long)] + ai_model: Option, + + /// AI API URL (e.g. "http://localhost:11434/v1" for Ollama) + #[arg(long)] + ai_api_url: Option, + + /// Slack webhook URL for alert notifications + #[arg(long)] + slack_webhook: Option, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[test] + fn test_no_subcommand_defaults_to_none() { + let cli = Cli::parse_from(["stackdog"]); + assert!(cli.command.is_none(), "No subcommand should yield None (default to serve)"); + } + + #[test] + fn test_serve_subcommand() { + let cli = Cli::parse_from(["stackdog", "serve"]); + assert!(matches!(cli.command, Some(Command::Serve))); + } + + #[test] + fn test_sniff_subcommand_defaults() { + let cli = Cli::parse_from(["stackdog", "sniff"]); + match cli.command { + Some(Command::Sniff { once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook }) => { + assert!(!once); + assert!(!consume); + assert_eq!(output, "./stackdog-logs/"); + assert!(sources.is_none()); + assert_eq!(interval, 30); + assert!(ai_provider.is_none()); + assert!(ai_model.is_none()); + assert!(ai_api_url.is_none()); + assert!(slack_webhook.is_none()); + } + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_once_flag() { + let cli = Cli::parse_from(["stackdog", "sniff", "--once"]); + match cli.command { + Some(Command::Sniff { once, .. }) => assert!(once), + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_consume_flag() { + let cli = Cli::parse_from(["stackdog", "sniff", "--consume"]); + match cli.command { + Some(Command::Sniff { consume, .. }) => assert!(consume), + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_all_options() { + let cli = Cli::parse_from([ + "stackdog", "sniff", + "--once", + "--consume", + "--output", "/tmp/logs/", + "--sources", "/var/log/syslog,/var/log/auth.log", + "--interval", "60", + "--ai-provider", "openai", + "--ai-model", "gpt-4o-mini", + "--ai-api-url", "https://api.openai.com/v1", + "--slack-webhook", "https://hooks.slack.com/services/T/B/xxx", + ]); + match cli.command { + Some(Command::Sniff { once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook }) => { + assert!(once); + assert!(consume); + assert_eq!(output, "/tmp/logs/"); + assert_eq!(sources.unwrap(), "/var/log/syslog,/var/log/auth.log"); + assert_eq!(interval, 60); + assert_eq!(ai_provider.unwrap(), "openai"); + assert_eq!(ai_model.unwrap(), "gpt-4o-mini"); + assert_eq!(ai_api_url.unwrap(), "https://api.openai.com/v1"); + assert_eq!(slack_webhook.unwrap(), "https://hooks.slack.com/services/T/B/xxx"); + } + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_candle_provider() { + let cli = Cli::parse_from(["stackdog", "sniff", "--ai-provider", "candle"]); + match cli.command { + Some(Command::Sniff { ai_provider, .. }) => { + assert_eq!(ai_provider.unwrap(), "candle"); + } + _ => panic!("Expected Sniff command"), + } + } + + #[test] + fn test_sniff_with_ollama_provider_and_model() { + let cli = Cli::parse_from([ + "stackdog", "sniff", + "--once", + "--ai-provider", "ollama", + "--ai-model", "qwen2.5-coder:latest", + ]); + match cli.command { + Some(Command::Sniff { ai_provider, ai_model, .. }) => { + assert_eq!(ai_provider.unwrap(), "ollama"); + assert_eq!(ai_model.unwrap(), "qwen2.5-coder:latest"); + } + _ => panic!("Expected Sniff command"), + } + } +} diff --git a/src/collectors/ebpf/container.rs b/src/collectors/ebpf/container.rs index 9cd568c..98de118 100644 --- a/src/collectors/ebpf/container.rs +++ b/src/collectors/ebpf/container.rs @@ -196,16 +196,16 @@ mod tests { #[test] fn test_parse_docker_cgroup() { - let cgroup = "12:memory:/docker/abc123def456789012345678901234567890"; + let cgroup = "12:memory:/docker/abc123def456abc123def456abc123def456abc123def456abc123def456abcd"; let result = ContainerDetector::parse_container_from_cgroup(cgroup); - assert_eq!(result, Some("abc123def456789012345678901234567890".to_string())); + assert_eq!(result, Some("abc123def456abc123def456abc123def456abc123def456abc123def456abcd".to_string())); } - + #[test] fn test_parse_kubernetes_cgroup() { - let cgroup = "11:cpu:/kubepods/pod123/def456abc789012345678901234567890"; + let cgroup = "11:cpu:/kubepods/pod123/def456abc123def456abc123def456abc123def456abc123def456abc123def4"; let result = ContainerDetector::parse_container_from_cgroup(cgroup); - assert_eq!(result, Some("def456abc789012345678901234567890".to_string())); + assert_eq!(result, Some("def456abc123def456abc123def456abc123def456abc123def456abc123def4".to_string())); } #[test] @@ -215,6 +215,7 @@ mod tests { assert_eq!(result, None); } + #[cfg(target_os = "linux")] #[test] fn test_validate_valid_container_id() { let detector = ContainerDetector::new().unwrap(); @@ -226,6 +227,7 @@ mod tests { assert!(detector.validate_container_id("abc123def456")); } + #[cfg(target_os = "linux")] #[test] fn test_validate_invalid_container_id() { let detector = ContainerDetector::new().unwrap(); diff --git a/src/collectors/ebpf/enrichment.rs b/src/collectors/ebpf/enrichment.rs index 00df2a6..fcbde6c 100644 --- a/src/collectors/ebpf/enrichment.rs +++ b/src/collectors/ebpf/enrichment.rs @@ -133,7 +133,7 @@ pub fn normalize_timestamp(ts: chrono::DateTime) -> chrono::DateTim #[cfg(test)] mod tests { use super::*; - + use chrono::Utc; #[test] fn test_enricher_creation() { let enricher = EventEnricher::new(); diff --git a/src/collectors/ebpf/loader.rs b/src/collectors/ebpf/loader.rs index 516b7d5..5838f1d 100644 --- a/src/collectors/ebpf/loader.rs +++ b/src/collectors/ebpf/loader.rs @@ -97,12 +97,15 @@ impl EbpfLoader { if _bytes.is_empty() { return Err(LoadError::LoadFailed("Empty program bytes".to_string())); } - - // TODO: Implement actual loading when eBPF programs are ready - // For now, this is a stub that will be implemented in TASK-004 + + let bpf = aya::Bpf::load(_bytes) + .map_err(|e| LoadError::LoadFailed(e.to_string()))?; + self.bpf = Some(bpf); + + log::info!("eBPF program loaded ({} bytes)", _bytes.len()); Ok(()) } - + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { Err(LoadError::NotLinux) @@ -132,23 +135,80 @@ impl EbpfLoader { pub fn attach_program(&mut self, _program_name: &str) -> Result<(), LoadError> { #[cfg(all(target_os = "linux", feature = "ebpf"))] { - // TODO: Implement actual attachment - // For now, just mark as attached + let (category, tp_name) = program_to_tracepoint(_program_name) + .ok_or_else(|| LoadError::ProgramNotFound( + format!("No tracepoint mapping for '{}'", _program_name) + ))?; + + let bpf = self.bpf.as_mut() + .ok_or_else(|| LoadError::LoadFailed( + "No eBPF program loaded; call load_program_from_bytes first".to_string() + ))?; + + let prog: &mut aya::programs::TracePoint = bpf + .program_mut(_program_name) + .ok_or_else(|| LoadError::ProgramNotFound(_program_name.to_string()))? + .try_into() + .map_err(|e: aya::programs::ProgramError| LoadError::AttachFailed(e.to_string()))?; + + prog.load() + .map_err(|e| LoadError::AttachFailed(format!("load '{}': {}", _program_name, e)))?; + + prog.attach(category, tp_name) + .map_err(|e| LoadError::AttachFailed( + format!("attach '{}/{}': {}", category, tp_name, e) + ))?; + self.loaded_programs.insert( _program_name.to_string(), - ProgramInfo { - name: _program_name.to_string(), - attached: true, - }, + ProgramInfo { name: _program_name.to_string(), attached: true }, ); + + log::info!("eBPF program '{}' attached to {}/{}", _program_name, category, tp_name); Ok(()) } - + + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] + { + Err(LoadError::NotLinux) + } + } + + /// Attach all known syscall tracepoint programs + pub fn attach_all_programs(&mut self) -> Result<(), LoadError> { + #[cfg(all(target_os = "linux", feature = "ebpf"))] + { + for name in &["trace_execve", "trace_connect", "trace_openat", "trace_ptrace"] { + if let Err(e) = self.attach_program(name) { + log::warn!("Failed to attach '{}': {}", name, e); + } + } + Ok(()) + } + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { Err(LoadError::NotLinux) } } + + /// Extract the EVENTS ring buffer map from the loaded eBPF program. + /// Must be called after load_program_from_bytes and before the Bpf object is dropped. + #[cfg(all(target_os = "linux", feature = "ebpf"))] + pub fn take_ring_buf(&mut self) -> Result, LoadError> { + let bpf = self.bpf.as_mut() + .ok_or_else(|| LoadError::LoadFailed( + "No eBPF program loaded".to_string() + ))?; + + let map = bpf.take_map("EVENTS") + .ok_or_else(|| LoadError::LoadFailed( + "EVENTS ring buffer map not found in eBPF program".to_string() + ))?; + + aya::maps::RingBuf::try_from(map) + .map_err(|e| LoadError::LoadFailed(format!("Failed to create ring buffer: {}", e))) + } /// Detach a program pub fn detach_program(&mut self, program_name: &str) -> Result<(), LoadError> { @@ -201,8 +261,24 @@ impl EbpfLoader { } impl Default for EbpfLoader { - fn default() -> Result { - Self::new() + fn default() -> Self { + Self { + #[cfg(all(target_os = "linux", feature = "ebpf"))] + bpf: None, + loaded_programs: HashMap::new(), + kernel_version: None, + } + } +} + +/// Map program name to its tracepoint (category, name) for aya attachment. +fn program_to_tracepoint(name: &str) -> Option<(&'static str, &'static str)> { + match name { + "trace_execve" => Some(("syscalls", "sys_enter_execve")), + "trace_connect" => Some(("syscalls", "sys_enter_connect")), + "trace_openat" => Some(("syscalls", "sys_enter_openat")), + "trace_ptrace" => Some(("syscalls", "sys_enter_ptrace")), + _ => None, } } diff --git a/src/collectors/ebpf/ring_buffer.rs b/src/collectors/ebpf/ring_buffer.rs index 1983a68..9c25b01 100644 --- a/src/collectors/ebpf/ring_buffer.rs +++ b/src/collectors/ebpf/ring_buffer.rs @@ -59,6 +59,11 @@ impl EventRingBuffer { self.capacity } + /// View events without consuming them + pub fn events(&self) -> &[SyscallEvent] { + &self.buffer + } + /// Clear the buffer pub fn clear(&mut self) { self.buffer.clear(); diff --git a/src/collectors/ebpf/syscall_monitor.rs b/src/collectors/ebpf/syscall_monitor.rs index 5f4828f..df92490 100644 --- a/src/collectors/ebpf/syscall_monitor.rs +++ b/src/collectors/ebpf/syscall_monitor.rs @@ -12,7 +12,10 @@ use crate::collectors::ebpf::container::ContainerDetector; pub struct SyscallMonitor { #[cfg(all(target_os = "linux", feature = "ebpf"))] loader: Option, - + + #[cfg(all(target_os = "linux", feature = "ebpf"))] + ring_buf: Option>, + running: bool, event_buffer: EventRingBuffer, enricher: EventEnricher, @@ -34,6 +37,7 @@ impl SyscallMonitor { Ok(Self { loader: Some(loader), + ring_buf: None, running: false, event_buffer: EventRingBuffer::with_capacity(8192), enricher, @@ -54,15 +58,36 @@ impl SyscallMonitor { if self.running { anyhow::bail!("Monitor is already running"); } - - // TODO: Actually start eBPF programs in TASK-004 - // For now, just mark as running + + if let Some(loader) = &mut self.loader { + let ebpf_path = "target/bpfel-unknown-none/release/stackdog"; + match loader.load_program_from_file(ebpf_path) { + Ok(()) => { + loader.attach_all_programs().unwrap_or_else(|e| { + log::warn!("Some eBPF programs failed to attach: {}", e); + }); + match loader.take_ring_buf() { + Ok(rb) => { self.ring_buf = Some(rb); } + Err(e) => { log::warn!("Failed to get eBPF ring buffer: {}", e); } + } + } + Err(e) => { + log::warn!( + "eBPF program not found at '{}': {}. \ + Running without kernel event collection — \ + build the eBPF crate first with `cargo build --release` \ + in the ebpf/ directory.", + ebpf_path, e + ); + } + } + } + self.running = true; - log::info!("Syscall monitor started"); Ok(()) } - + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { anyhow::bail!("SyscallMonitor is only available on Linux"); @@ -73,7 +98,10 @@ impl SyscallMonitor { pub fn stop(&mut self) -> Result<()> { self.running = false; self.event_buffer.clear(); - + #[cfg(all(target_os = "linux", feature = "ebpf"))] + { + self.ring_buf = None; + } log::info!("Syscall monitor stopped"); Ok(()) } @@ -90,25 +118,39 @@ impl SyscallMonitor { if !self.running { return Vec::new(); } - - // TODO: Actually poll eBPF ring buffer in TASK-004 - // For now, drain from internal buffer + + // Drain the eBPF ring buffer into the staging buffer + if let Some(rb) = &mut self.ring_buf { + while let Some(item) = rb.next() { + let bytes: &[u8] = &item; + if bytes.len() >= std::mem::size_of::() { + // SAFETY: We verified the byte length matches the struct size, + // and EbpfSyscallEvent is #[repr(C)] with no padding surprises. + let raw: super::types::EbpfSyscallEvent = unsafe { + std::ptr::read_unaligned( + bytes.as_ptr() as *const super::types::EbpfSyscallEvent + ) + }; + self.event_buffer.push(raw.to_syscall_event()); + } + } + } + + // Drain the staging buffer and enrich with /proc info let mut events = self.event_buffer.drain(); - - // Enrich events for event in &mut events { let _ = self.enricher.enrich(event); } - + events } - + #[cfg(not(all(target_os = "linux", feature = "ebpf")))] { Vec::new() } } - + /// Get events without consuming them pub fn peek_events(&self) -> &[SyscallEvent] { self.event_buffer.events() diff --git a/src/collectors/ebpf/types.rs b/src/collectors/ebpf/types.rs index f8ef26a..6e97d28 100644 --- a/src/collectors/ebpf/types.rs +++ b/src/collectors/ebpf/types.rs @@ -27,7 +27,7 @@ pub struct EbpfSyscallEvent { /// Event data union #[repr(C)] -#[derive(Debug, Clone, Copy)] +#[derive(Clone, Copy)] pub union EbpfEventData { /// execve data pub execve: ExecveData, @@ -41,6 +41,14 @@ pub union EbpfEventData { pub raw: [u8; 128], } +impl std::fmt::Debug for EbpfEventData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // SAFETY: raw is always a valid field in any union variant + let raw = unsafe { self.raw }; + write!(f, "EbpfEventData {{ raw: {:?} }}", &raw[..]) + } +} + impl Default for EbpfEventData { fn default() -> Self { Self { @@ -51,7 +59,7 @@ impl Default for EbpfEventData { /// execve-specific data #[repr(C)] -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] pub struct ExecveData { /// Filename length pub filename_len: u32, @@ -61,6 +69,12 @@ pub struct ExecveData { pub argc: u32, } +impl Default for ExecveData { + fn default() -> Self { + Self { filename_len: 0, filename: [0u8; 128], argc: 0 } + } +} + /// connect-specific data #[repr(C)] #[derive(Debug, Clone, Copy, Default)] @@ -75,7 +89,7 @@ pub struct ConnectData { /// openat-specific data #[repr(C)] -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] pub struct OpenatData { /// File path length pub path_len: u32, @@ -85,6 +99,12 @@ pub struct OpenatData { pub flags: u32, } +impl Default for OpenatData { + fn default() -> Self { + Self { path_len: 0, path: [0u8; 256], flags: 0 } + } +} + /// ptrace-specific data #[repr(C)] #[derive(Debug, Clone, Copy, Default)] diff --git a/src/database/connection.rs b/src/database/connection.rs index d64ab39..d98d619 100644 --- a/src/database/connection.rs +++ b/src/database/connection.rs @@ -108,6 +108,38 @@ pub fn init_database(pool: &DbPool) -> Result<()> { let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_containers_status ON containers_cache(status)", []); let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_containers_name ON containers_cache(name)", []); + + // Create log_sources table + conn.execute( + "CREATE TABLE IF NOT EXISTS log_sources ( + id TEXT PRIMARY KEY, + source_type TEXT NOT NULL, + path_or_id TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + discovered_at TEXT NOT NULL, + last_read_position INTEGER DEFAULT 0 + )", + [], + )?; + + // Create log_summaries table + conn.execute( + "CREATE TABLE IF NOT EXISTS log_summaries ( + id TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + summary_text TEXT NOT NULL, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + total_entries INTEGER DEFAULT 0, + error_count INTEGER DEFAULT 0, + warning_count INTEGER DEFAULT 0, + created_at TEXT NOT NULL + )", + [], + )?; + + let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_log_sources_type ON log_sources(source_type)", []); + let _ = conn.execute("CREATE INDEX IF NOT EXISTS idx_log_summaries_source ON log_summaries(source_id)", []); Ok(()) } diff --git a/src/database/mod.rs b/src/database/mod.rs index e9bbe45..c8fa512 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -7,3 +7,6 @@ pub mod repositories; pub use connection::{create_pool, init_database, DbPool}; pub use models::*; pub use repositories::alerts::*; + +/// Marker struct for module tests +pub struct DatabaseMarker; diff --git a/src/database/repositories/log_sources.rs b/src/database/repositories/log_sources.rs new file mode 100644 index 0000000..70e45fe --- /dev/null +++ b/src/database/repositories/log_sources.rs @@ -0,0 +1,308 @@ +//! Log sources repository using rusqlite +//! +//! Persists discovered log sources and AI summaries, following +//! the same pattern as the alerts repository. + +use rusqlite::params; +use anyhow::Result; +use crate::database::connection::DbPool; +use crate::sniff::discovery::{LogSource, LogSourceType}; +use chrono::Utc; + +/// Create or update a log source (upsert by path_or_id) +pub fn upsert_log_source(pool: &DbPool, source: &LogSource) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "INSERT INTO log_sources (id, source_type, path_or_id, name, discovered_at, last_read_position) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(path_or_id) DO UPDATE SET + name = excluded.name, + source_type = excluded.source_type", + params![ + source.id, + source.source_type.to_string(), + source.path_or_id, + source.name, + source.discovered_at.to_rfc3339(), + source.last_read_position as i64, + ], + )?; + Ok(()) +} + +/// List all registered log sources +pub fn list_log_sources(pool: &DbPool) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, source_type, path_or_id, name, discovered_at, last_read_position + FROM log_sources ORDER BY discovered_at DESC" + )?; + + let sources = stmt.query_map([], |row| { + let source_type_str: String = row.get(1)?; + let discovered_str: String = row.get(4)?; + let pos: i64 = row.get(5)?; + Ok(LogSource { + id: row.get(0)?, + source_type: LogSourceType::from_str(&source_type_str), + path_or_id: row.get(2)?, + name: row.get(3)?, + discovered_at: chrono::DateTime::parse_from_rfc3339(&discovered_str) + .map(|dt| dt.with_timezone(&Utc)) + .unwrap_or_else(|_| Utc::now()), + last_read_position: pos as u64, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + + Ok(sources) +} + +/// Get a log source by its path or container ID +pub fn get_log_source_by_path(pool: &DbPool, path_or_id: &str) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, source_type, path_or_id, name, discovered_at, last_read_position + FROM log_sources WHERE path_or_id = ?" + )?; + + let result = stmt.query_row(params![path_or_id], |row| { + let source_type_str: String = row.get(1)?; + let discovered_str: String = row.get(4)?; + let pos: i64 = row.get(5)?; + Ok(LogSource { + id: row.get(0)?, + source_type: LogSourceType::from_str(&source_type_str), + path_or_id: row.get(2)?, + name: row.get(3)?, + discovered_at: chrono::DateTime::parse_from_rfc3339(&discovered_str) + .map(|dt| dt.with_timezone(&Utc)) + .unwrap_or_else(|_| Utc::now()), + last_read_position: pos as u64, + }) + }); + + match result { + Ok(source) => Ok(Some(source)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(anyhow::anyhow!("Database error: {}", e)), + } +} + +/// Update the read position for a log source +pub fn update_read_position(pool: &DbPool, path_or_id: &str, position: u64) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "UPDATE log_sources SET last_read_position = ?1 WHERE path_or_id = ?2", + params![position as i64, path_or_id], + )?; + Ok(()) +} + +/// Delete a log source +pub fn delete_log_source(pool: &DbPool, path_or_id: &str) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "DELETE FROM log_sources WHERE path_or_id = ?", + params![path_or_id], + )?; + Ok(()) +} + +/// Store a log summary +pub fn create_log_summary( + pool: &DbPool, + source_id: &str, + summary_text: &str, + period_start: &str, + period_end: &str, + total_entries: i64, + error_count: i64, + warning_count: i64, +) -> Result { + let conn = pool.get()?; + let id = uuid::Uuid::new_v4().to_string(); + let now = Utc::now().to_rfc3339(); + + conn.execute( + "INSERT INTO log_summaries (id, source_id, summary_text, period_start, period_end, + total_entries, error_count, warning_count, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![id, source_id, summary_text, period_start, period_end, + total_entries, error_count, warning_count, now], + )?; + + Ok(id) +} + +/// List summaries for a source +pub fn list_summaries_for_source(pool: &DbPool, source_id: &str) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, source_id, summary_text, period_start, period_end, + total_entries, error_count, warning_count, created_at + FROM log_summaries WHERE source_id = ? ORDER BY created_at DESC" + )?; + + let rows = stmt.query_map(params![source_id], |row| { + Ok(LogSummaryRow { + id: row.get(0)?, + source_id: row.get(1)?, + summary_text: row.get(2)?, + period_start: row.get(3)?, + period_end: row.get(4)?, + total_entries: row.get(5)?, + error_count: row.get(6)?, + warning_count: row.get(7)?, + created_at: row.get(8)?, + }) + })? + .filter_map(|r| r.ok()) + .collect(); + + Ok(rows) +} + +/// Database row for a log summary +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LogSummaryRow { + pub id: String, + pub source_id: String, + pub summary_text: String, + pub period_start: String, + pub period_end: String, + pub total_entries: i64, + pub error_count: i64, + pub warning_count: i64, + pub created_at: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::database::connection::{create_pool, init_database}; + + fn setup_test_db() -> DbPool { + let pool = create_pool(":memory:").unwrap(); + init_database(&pool).unwrap(); + pool + } + + #[test] + fn test_upsert_and_list_log_sources() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/test.log".into(), + "test.log".into(), + ); + + upsert_log_source(&pool, &source).unwrap(); + let sources = list_log_sources(&pool).unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].path_or_id, "/var/log/test.log"); + assert_eq!(sources[0].name, "test.log"); + } + + #[test] + fn test_upsert_deduplicates_by_path() { + let pool = setup_test_db(); + let source1 = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog-v1".into(), + ); + let source2 = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog-v2".into(), + ); + + upsert_log_source(&pool, &source1).unwrap(); + upsert_log_source(&pool, &source2).unwrap(); + + let sources = list_log_sources(&pool).unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].name, "syslog-v2"); + } + + #[test] + fn test_get_log_source_by_path() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::DockerContainer, + "container-abc123".into(), + "docker:myapp".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + + let found = get_log_source_by_path(&pool, "container-abc123").unwrap(); + assert!(found.is_some()); + assert_eq!(found.unwrap().name, "docker:myapp"); + + let not_found = get_log_source_by_path(&pool, "nonexistent").unwrap(); + assert!(not_found.is_none()); + } + + #[test] + fn test_update_read_position() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::CustomFile, + "/tmp/app.log".into(), + "app.log".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + + update_read_position(&pool, "/tmp/app.log", 4096).unwrap(); + + let updated = get_log_source_by_path(&pool, "/tmp/app.log").unwrap().unwrap(); + assert_eq!(updated.last_read_position, 4096); + } + + #[test] + fn test_delete_log_source() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/test.log".into(), + "test.log".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + assert_eq!(list_log_sources(&pool).unwrap().len(), 1); + + delete_log_source(&pool, "/var/log/test.log").unwrap(); + assert_eq!(list_log_sources(&pool).unwrap().len(), 0); + } + + #[test] + fn test_create_and_list_summaries() { + let pool = setup_test_db(); + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog".into(), + ); + upsert_log_source(&pool, &source).unwrap(); + + let summary_id = create_log_summary( + &pool, + &source.id, + "System running normally. 3 warnings about disk space.", + "2026-03-30T12:00:00Z", + "2026-03-30T13:00:00Z", + 500, + 0, + 3, + ).unwrap(); + + assert!(!summary_id.is_empty()); + + let summaries = list_summaries_for_source(&pool, &source.id).unwrap(); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].total_entries, 500); + assert_eq!(summaries[0].warning_count, 3); + assert!(summaries[0].summary_text.contains("disk space")); + } +} diff --git a/src/database/repositories/mod.rs b/src/database/repositories/mod.rs index 92b469d..8f790f5 100644 --- a/src/database/repositories/mod.rs +++ b/src/database/repositories/mod.rs @@ -1,6 +1,6 @@ //! Database repositories pub mod alerts; -// TODO: Add threats and containers repositories +pub mod log_sources; pub use alerts::*; diff --git a/src/docker/client.rs b/src/docker/client.rs index 6bf03e3..751fe14 100644 --- a/src/docker/client.rs +++ b/src/docker/client.rs @@ -29,7 +29,7 @@ impl DockerClient { /// List all containers pub async fn list_containers(&self, all: bool) -> Result> { - let options = Some(ListContainersOptions { + let options: Option> = Some(ListContainersOptions { all, size: false, ..Default::default() @@ -85,7 +85,7 @@ impl DockerClient { /// Quarantine a container (disconnect from all networks) pub async fn quarantine_container(&self, container_id: &str) -> Result<()> { // List all networks - let networks: Vec = self.client + let networks: Vec = self.client .list_networks(None::>) .await .context("Failed to list networks")?; @@ -104,7 +104,7 @@ impl DockerClient { }; let _ = self.client - .disconnect_network(&name, Some(options)) + .disconnect_network(&name, options) .await; } } diff --git a/src/lib.rs b/src/lib.rs index 622b684..8a64c1d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,8 +46,7 @@ pub mod models; #[cfg(target_os = "linux")] pub mod firewall; -// Security modules - Collectors -#[cfg(target_os = "linux")] +// Security modules - Collectors (cross-platform; Linux-specific internals are gated within) pub mod collectors; // Optional modules @@ -56,10 +55,14 @@ pub mod response; pub mod correlator; pub mod baselines; pub mod database; +pub mod docker; // Configuration pub mod config; +// Log sniffing +pub mod sniff; + // Re-export commonly used types pub use events::syscall::{SyscallEvent, SyscallType}; pub use events::security::{SecurityEvent, NetworkEvent, ContainerEvent, AlertEvent}; @@ -74,7 +77,6 @@ pub use alerting::{NotificationChannel, NotificationConfig}; pub use firewall::{QuarantineManager, QuarantineState}; #[cfg(target_os = "linux")] pub use firewall::{ResponseAction, ResponseChain, ResponseExecutor, ResponseType}; -#[cfg(target_os = "linux")] pub use collectors::{EbpfLoader, SyscallMonitor}; // Rules diff --git a/src/main.rs b/src/main.rs index 33ccb20..4bb0619 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,26 +22,47 @@ mod config; mod api; mod database; mod docker; +mod events; +mod rules; +mod alerting; +mod models; +mod cli; +mod sniff; use std::{io, env}; use actix_web::{HttpServer, App, web}; use actix_cors::Cors; +use clap::Parser; use tracing::{Level, info}; use tracing_subscriber::FmtSubscriber; use database::{create_pool, init_database}; +use cli::{Cli, Command}; #[actix_rt::main] async fn main() -> io::Result<()> { // Load environment dotenv::dotenv().expect("Could not read .env file"); + // Parse CLI arguments + let cli = Cli::parse(); + // Setup logging - env::set_var("RUST_LOG", "stackdog=info,actix_web=info"); + // Only set default RUST_LOG if user hasn't configured it + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "stackdog=info,actix_web=info"); + } env_logger::init(); - // Setup tracing + // Setup tracing — respect RUST_LOG for level + let max_level = if env::var("RUST_LOG").map(|v| v.contains("debug")).unwrap_or(false) { + Level::DEBUG + } else if env::var("RUST_LOG").map(|v| v.contains("trace")).unwrap_or(false) { + Level::TRACE + } else { + Level::INFO + }; let subscriber = FmtSubscriber::builder() - .with_max_level(Level::INFO) + .with_max_level(max_level) .finish(); tracing::subscriber::set_global_default(subscriber) .expect("setting default subscriber failed"); @@ -49,8 +70,17 @@ async fn main() -> io::Result<()> { info!("🐕 Stackdog Security starting..."); info!("Platform: {}", std::env::consts::OS); info!("Architecture: {}", std::env::consts::ARCH); - - // Display configuration + + match cli.command { + Some(Command::Sniff { once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook }) => { + run_sniff(once, consume, output, sources, interval, ai_provider, ai_model, ai_api_url, slack_webhook).await + } + // Default: serve (backward compatible) + Some(Command::Serve) | None => run_serve().await, + } +} + +async fn run_serve() -> io::Result<()> { let app_host = env::var("APP_HOST").unwrap_or_else(|_| "0.0.0.0".to_string()); let app_port = env::var("APP_PORT").unwrap_or_else(|_| "5000".to_string()); let database_url = env::var("DATABASE_URL").unwrap_or_else(|_| "./stackdog.db".to_string()); @@ -78,6 +108,9 @@ async fn main() -> io::Result<()> { info!(" POST /api/containers/:id/quar - Quarantine container"); info!(" GET /api/threats - List threats"); info!(" GET /api/threats/statistics - Threat statistics"); + info!(" GET /api/logs/sources - List log sources"); + info!(" POST /api/logs/sources - Add log source"); + info!(" GET /api/logs/summaries - List AI summaries"); info!(" WS /ws - WebSocket for real-time updates"); info!(""); info!("Web Dashboard: http://{}:{}", app_host, app_port); @@ -99,3 +132,46 @@ async fn main() -> io::Result<()> { .run() .await } + +async fn run_sniff( + once: bool, + consume: bool, + output: String, + sources: Option, + interval: u64, + ai_provider: Option, + ai_model: Option, + ai_api_url: Option, + slack_webhook: Option, +) -> io::Result<()> { + let config = sniff::config::SniffConfig::from_env_and_args( + once, + consume, + &output, + sources.as_deref(), + interval, + ai_provider.as_deref(), + ai_model.as_deref(), + ai_api_url.as_deref(), + slack_webhook.as_deref(), + ); + + info!("🔍 Stackdog Sniff starting..."); + info!("Mode: {}", if config.once { "one-shot" } else { "continuous" }); + info!("Consume: {}", config.consume); + info!("Output: {}", config.output_dir.display()); + info!("Interval: {}s", config.interval_secs); + info!("AI Provider: {:?}", config.ai_provider); + info!("AI Model: {}", config.ai_model); + info!("AI API URL: {}", config.ai_api_url); + if config.slack_webhook.is_some() { + info!("Slack: configured ✓"); + } + + let orchestrator = sniff::SniffOrchestrator::new(config) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + orchestrator.run().await + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) +} + diff --git a/src/sniff/analyzer.rs b/src/sniff/analyzer.rs new file mode 100644 index 0000000..5eee30e --- /dev/null +++ b/src/sniff/analyzer.rs @@ -0,0 +1,639 @@ +//! AI-powered log analysis engine +//! +//! Provides log summarization and anomaly detection via two backends: +//! - OpenAI-compatible API (works with OpenAI, Ollama, vLLM, etc.) +//! - Local Candle inference (requires `ml` feature) + +use anyhow::{Result, Context}; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::sniff::reader::LogEntry; + +/// Summary produced by AI analysis of log entries +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogSummary { + pub source_id: String, + pub period_start: DateTime, + pub period_end: DateTime, + pub total_entries: usize, + pub summary_text: String, + pub error_count: usize, + pub warning_count: usize, + pub key_events: Vec, + pub anomalies: Vec, +} + +/// An anomaly detected in log entries +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogAnomaly { + pub description: String, + pub severity: AnomalySeverity, + pub sample_line: String, +} + +/// Severity of a detected anomaly +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum AnomalySeverity { + Low, + Medium, + High, + Critical, +} + +impl std::fmt::Display for AnomalySeverity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AnomalySeverity::Low => write!(f, "Low"), + AnomalySeverity::Medium => write!(f, "Medium"), + AnomalySeverity::High => write!(f, "High"), + AnomalySeverity::Critical => write!(f, "Critical"), + } + } +} + +/// Trait for AI-powered log analysis +#[async_trait] +pub trait LogAnalyzer: Send + Sync { + /// Summarize a batch of log entries + async fn summarize(&self, entries: &[LogEntry]) -> Result; +} + +/// OpenAI-compatible API backend (works with OpenAI, Ollama, vLLM, etc.) +pub struct OpenAiAnalyzer { + api_url: String, + api_key: Option, + model: String, + client: reqwest::Client, +} + +impl OpenAiAnalyzer { + pub fn new(api_url: String, api_key: Option, model: String) -> Self { + Self { + api_url, + api_key, + model, + client: reqwest::Client::new(), + } + } + + fn build_prompt(entries: &[LogEntry]) -> String { + let lines: Vec<&str> = entries.iter().map(|e| e.line.as_str()).collect(); + let log_block = lines.join("\n"); + + format!( + "Analyze these log entries and provide a JSON response with:\n\ + 1. \"summary\": A concise summary of what happened\n\ + 2. \"error_count\": Number of errors found\n\ + 3. \"warning_count\": Number of warnings found\n\ + 4. \"key_events\": Array of important events (max 5)\n\ + 5. \"anomalies\": Array of objects with \"description\", \"severity\" (Low/Medium/High/Critical), \"sample_line\"\n\n\ + Respond ONLY with valid JSON, no markdown.\n\n\ + Log entries:\n{}", log_block + ) + } +} + +/// Response structure from the LLM +#[derive(Debug, Deserialize)] +struct LlmAnalysis { + summary: Option, + error_count: Option, + warning_count: Option, + key_events: Option>, + anomalies: Option>, +} + +#[derive(Debug, Deserialize)] +struct LlmAnomaly { + description: Option, + severity: Option, + sample_line: Option, +} + +/// OpenAI chat completion response +#[derive(Debug, Deserialize)] +struct ChatCompletionResponse { + choices: Vec, +} + +#[derive(Debug, Deserialize)] +struct ChatChoice { + message: ChatMessage, +} + +#[derive(Debug, Deserialize, Serialize)] +struct ChatMessage { + role: String, + content: String, +} + +/// Extract JSON from LLM response, handling markdown fences, preamble text, etc. +fn extract_json(content: &str) -> &str { + let trimmed = content.trim(); + + // Try ```json ... ``` fence + if let Some(start) = trimmed.find("```json") { + let after_fence = &trimmed[start + 7..]; + if let Some(end) = after_fence.find("```") { + return after_fence[..end].trim(); + } + } + + // Try ``` ... ``` fence (no language tag) + if let Some(start) = trimmed.find("```") { + let after_fence = &trimmed[start + 3..]; + if let Some(end) = after_fence.find("```") { + return after_fence[..end].trim(); + } + } + + // Try to find raw JSON object + if let Some(start) = trimmed.find('{') { + if let Some(end) = trimmed.rfind('}') { + if end > start { + return &trimmed[start..=end]; + } + } + } + + trimmed +} + +/// Parse LLM severity string to enum +fn parse_severity(s: &str) -> AnomalySeverity { + match s.to_lowercase().as_str() { + "critical" => AnomalySeverity::Critical, + "high" => AnomalySeverity::High, + "medium" => AnomalySeverity::Medium, + _ => AnomalySeverity::Low, + } +} + +/// Parse the LLM JSON response into a LogSummary +fn parse_llm_response(source_id: &str, entries: &[LogEntry], raw_json: &str) -> Result { + log::debug!("Parsing LLM response ({} bytes) for source {}", raw_json.len(), source_id); + log::trace!("Raw LLM response:\n{}", raw_json); + + let analysis: LlmAnalysis = serde_json::from_str(raw_json) + .context(format!( + "Failed to parse LLM response as JSON. Response starts with: {}", + &raw_json[..raw_json.len().min(200)] + ))?; + + log::debug!( + "LLM analysis parsed — summary: {:?}, errors: {:?}, warnings: {:?}, anomalies: {}", + analysis.summary.as_deref().map(|s| &s[..s.len().min(80)]), + analysis.error_count, + analysis.warning_count, + analysis.anomalies.as_ref().map(|a| a.len()).unwrap_or(0), + ); + + let anomalies = analysis.anomalies.unwrap_or_default() + .into_iter() + .map(|a| LogAnomaly { + description: a.description.unwrap_or_default(), + severity: parse_severity(&a.severity.unwrap_or_default()), + sample_line: a.sample_line.unwrap_or_default(), + }) + .collect(); + + let (start, end) = entry_time_range(entries); + + Ok(LogSummary { + source_id: source_id.to_string(), + period_start: start, + period_end: end, + total_entries: entries.len(), + summary_text: analysis.summary.unwrap_or_else(|| "No summary available".into()), + error_count: analysis.error_count.unwrap_or(0), + warning_count: analysis.warning_count.unwrap_or(0), + key_events: analysis.key_events.unwrap_or_default(), + anomalies, + }) +} + +/// Compute time range from entries +fn entry_time_range(entries: &[LogEntry]) -> (DateTime, DateTime) { + if entries.is_empty() { + let now = Utc::now(); + return (now, now); + } + let start = entries.iter().map(|e| e.timestamp).min().unwrap_or_else(Utc::now); + let end = entries.iter().map(|e| e.timestamp).max().unwrap_or_else(Utc::now); + (start, end) +} + +#[async_trait] +impl LogAnalyzer for OpenAiAnalyzer { + async fn summarize(&self, entries: &[LogEntry]) -> Result { + if entries.is_empty() { + log::debug!("OpenAiAnalyzer: no entries to analyze, returning empty summary"); + return Ok(LogSummary { + source_id: String::new(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 0, + summary_text: "No log entries to analyze".into(), + error_count: 0, + warning_count: 0, + key_events: Vec::new(), + anomalies: Vec::new(), + }); + } + + let prompt = Self::build_prompt(entries); + let source_id = &entries[0].source_id; + + log::debug!( + "Sending {} entries to AI API (model: {}, url: {})", + entries.len(), self.model, self.api_url + ); + log::trace!("Prompt:\n{}", prompt); + + let request_body = serde_json::json!({ + "model": self.model, + "messages": [ + { + "role": "system", + "content": "You are a log analysis assistant. Analyze logs and return structured JSON." + }, + { + "role": "user", + "content": prompt + } + ], + "temperature": 0.1 + }); + + let url = format!("{}/chat/completions", self.api_url.trim_end_matches('/')); + log::debug!("POST {}", url); + + let mut req = self.client.post(&url) + .header("Content-Type", "application/json"); + + if let Some(ref key) = self.api_key { + log::debug!("Using API key: {}...{}", &key[..key.len().min(4)], &key[key.len().saturating_sub(4)..]); + req = req.header("Authorization", format!("Bearer {}", key)); + } else { + log::debug!("No API key configured (using keyless access)"); + } + + let response = req + .json(&request_body) + .send() + .await + .context("Failed to send request to AI API")?; + + let status = response.status(); + log::debug!("AI API response status: {}", status); + + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + log::debug!("AI API error body: {}", body); + anyhow::bail!("AI API returned status {}: {}", status, body); + } + + let raw_body = response.text().await + .context("Failed to read AI API response body")?; + log::debug!("AI API response body ({} bytes)", raw_body.len()); + log::trace!("AI API raw response:\n{}", raw_body); + + let completion: ChatCompletionResponse = serde_json::from_str(&raw_body) + .context("Failed to parse AI API response as ChatCompletion")?; + + let content = completion.choices + .first() + .map(|c| c.message.content.clone()) + .unwrap_or_default(); + + log::debug!("LLM content ({} chars): {}", content.len(), &content[..content.len().min(200)]); + + // Extract JSON from response — LLMs often wrap in markdown code fences + let json_str = extract_json(&content); + log::debug!("Extracted JSON ({} chars)", json_str.len()); + + parse_llm_response(source_id, entries, json_str) + } +} + +/// Fallback local analyzer that uses pattern matching (no AI required) +pub struct PatternAnalyzer; + +impl PatternAnalyzer { + pub fn new() -> Self { + Self + } + + fn count_pattern(entries: &[LogEntry], patterns: &[&str]) -> usize { + entries.iter().filter(|e| { + let lower = e.line.to_lowercase(); + patterns.iter().any(|p| lower.contains(p)) + }).count() + } +} + +#[async_trait] +impl LogAnalyzer for PatternAnalyzer { + async fn summarize(&self, entries: &[LogEntry]) -> Result { + if entries.is_empty() { + log::debug!("PatternAnalyzer: no entries to analyze"); + return Ok(LogSummary { + source_id: String::new(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 0, + summary_text: "No log entries to analyze".into(), + error_count: 0, + warning_count: 0, + key_events: Vec::new(), + anomalies: Vec::new(), + }); + } + + let source_id = &entries[0].source_id; + let error_count = Self::count_pattern(entries, &["error", "err", "fatal", "panic", "exception"]); + let warning_count = Self::count_pattern(entries, &["warn", "warning"]); + let (start, end) = entry_time_range(entries); + + log::debug!( + "PatternAnalyzer [{}]: {} entries, {} errors, {} warnings", + source_id, entries.len(), error_count, warning_count + ); + + let mut anomalies = Vec::new(); + + // Detect error spikes + if error_count > entries.len() / 4 { + log::debug!( + "Error spike detected: {} errors / {} entries (threshold: >25%)", + error_count, entries.len() + ); + if let Some(sample) = entries.iter().find(|e| e.line.to_lowercase().contains("error")) { + anomalies.push(LogAnomaly { + description: format!("High error rate: {} errors in {} entries", error_count, entries.len()), + severity: AnomalySeverity::High, + sample_line: sample.line.clone(), + }); + } + } + + let summary_text = format!( + "{} log entries analyzed. {} errors, {} warnings detected.", + entries.len(), error_count, warning_count + ); + + Ok(LogSummary { + source_id: source_id.clone(), + period_start: start, + period_end: end, + total_entries: entries.len(), + summary_text, + error_count, + warning_count, + key_events: Vec::new(), + anomalies, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn make_entries(lines: &[&str]) -> Vec { + lines.iter().map(|line| LogEntry { + source_id: "test-source".into(), + timestamp: Utc::now(), + line: line.to_string(), + metadata: HashMap::new(), + }).collect() + } + + #[test] + fn test_anomaly_severity_display() { + assert_eq!(AnomalySeverity::Low.to_string(), "Low"); + assert_eq!(AnomalySeverity::Critical.to_string(), "Critical"); + } + + #[test] + fn test_parse_severity() { + assert_eq!(parse_severity("critical"), AnomalySeverity::Critical); + assert_eq!(parse_severity("High"), AnomalySeverity::High); + assert_eq!(parse_severity("MEDIUM"), AnomalySeverity::Medium); + assert_eq!(parse_severity("low"), AnomalySeverity::Low); + assert_eq!(parse_severity("unknown"), AnomalySeverity::Low); + } + + #[test] + fn test_build_prompt_contains_log_lines() { + let entries = make_entries(&["line 1", "line 2"]); + let prompt = OpenAiAnalyzer::build_prompt(&entries); + assert!(prompt.contains("line 1")); + assert!(prompt.contains("line 2")); + assert!(prompt.contains("JSON")); + } + + #[test] + fn test_parse_llm_response_valid() { + let entries = make_entries(&["test line"]); + let json = r#"{ + "summary": "System running normally", + "error_count": 0, + "warning_count": 1, + "key_events": ["Service started"], + "anomalies": [] + }"#; + + let summary = parse_llm_response("src-1", &entries, json).unwrap(); + assert_eq!(summary.source_id, "src-1"); + assert_eq!(summary.summary_text, "System running normally"); + assert_eq!(summary.error_count, 0); + assert_eq!(summary.warning_count, 1); + assert_eq!(summary.key_events.len(), 1); + assert!(summary.anomalies.is_empty()); + } + + #[test] + fn test_parse_llm_response_with_anomalies() { + let entries = make_entries(&["error: disk full"]); + let json = r#"{ + "summary": "Disk issue detected", + "error_count": 1, + "warning_count": 0, + "key_events": ["Disk full"], + "anomalies": [ + { + "description": "Disk full errors detected", + "severity": "Critical", + "sample_line": "error: disk full" + } + ] + }"#; + + let summary = parse_llm_response("src-1", &entries, json).unwrap(); + assert_eq!(summary.anomalies.len(), 1); + assert_eq!(summary.anomalies[0].severity, AnomalySeverity::Critical); + assert!(summary.anomalies[0].description.contains("Disk full")); + } + + #[test] + fn test_parse_llm_response_partial_fields() { + let entries = make_entries(&["line"]); + let json = r#"{"summary": "Minimal response"}"#; + + let summary = parse_llm_response("src-1", &entries, json).unwrap(); + assert_eq!(summary.summary_text, "Minimal response"); + assert_eq!(summary.error_count, 0); + assert!(summary.anomalies.is_empty()); + } + + #[test] + fn test_parse_llm_response_invalid_json() { + let entries = make_entries(&["line"]); + let result = parse_llm_response("src-1", &entries, "not json"); + assert!(result.is_err()); + } + + #[test] + fn test_extract_json_plain() { + let input = r#"{"summary": "ok"}"#; + assert_eq!(extract_json(input), input); + } + + #[test] + fn test_extract_json_markdown_fence() { + let input = "```json\n{\"summary\": \"ok\"}\n```"; + assert_eq!(extract_json(input), r#"{"summary": "ok"}"#); + } + + #[test] + fn test_extract_json_plain_fence() { + let input = "```\n{\"summary\": \"ok\"}\n```"; + assert_eq!(extract_json(input), r#"{"summary": "ok"}"#); + } + + #[test] + fn test_extract_json_with_preamble() { + let input = "Here is the analysis:\n{\"summary\": \"ok\", \"error_count\": 0}"; + assert_eq!(extract_json(input), r#"{"summary": "ok", "error_count": 0}"#); + } + + #[test] + fn test_extract_json_with_trailing_text() { + let input = "Sure! {\"summary\": \"ok\"} Hope this helps!"; + assert_eq!(extract_json(input), r#"{"summary": "ok"}"#); + } + + #[test] + fn test_entry_time_range_empty() { + let (start, end) = entry_time_range(&[]); + assert!(end >= start); + } + + #[test] + fn test_entry_time_range_multiple() { + let mut entries = make_entries(&["a", "b"]); + entries[0].timestamp = Utc::now() - chrono::Duration::hours(1); + let (start, end) = entry_time_range(&entries); + assert!(end > start); + } + + #[tokio::test] + async fn test_pattern_analyzer_empty() { + let analyzer = PatternAnalyzer::new(); + let summary = analyzer.summarize(&[]).await.unwrap(); + assert_eq!(summary.total_entries, 0); + assert!(summary.summary_text.contains("No log entries")); + } + + #[tokio::test] + async fn test_pattern_analyzer_counts_errors() { + let analyzer = PatternAnalyzer::new(); + let entries = make_entries(&[ + "INFO: started", + "ERROR: connection refused", + "WARN: disk space low", + "ERROR: timeout", + ]); + let summary = analyzer.summarize(&entries).await.unwrap(); + assert_eq!(summary.total_entries, 4); + assert_eq!(summary.error_count, 2); + assert_eq!(summary.warning_count, 1); + } + + #[tokio::test] + async fn test_pattern_analyzer_detects_error_spike() { + let analyzer = PatternAnalyzer::new(); + let entries = make_entries(&[ + "ERROR: fail 1", + "ERROR: fail 2", + "ERROR: fail 3", + "INFO: ok", + ]); + let summary = analyzer.summarize(&entries).await.unwrap(); + assert!(!summary.anomalies.is_empty()); + assert_eq!(summary.anomalies[0].severity, AnomalySeverity::High); + } + + #[tokio::test] + async fn test_pattern_analyzer_no_anomaly_when_low_errors() { + let analyzer = PatternAnalyzer::new(); + let entries = make_entries(&[ + "INFO: all good", + "INFO: running fine", + "INFO: healthy", + "ERROR: one blip", + ]); + let summary = analyzer.summarize(&entries).await.unwrap(); + assert!(summary.anomalies.is_empty()); + } + + #[test] + fn test_openai_analyzer_new() { + let analyzer = OpenAiAnalyzer::new( + "http://localhost:11434/v1".into(), + None, + "llama3".into(), + ); + assert_eq!(analyzer.api_url, "http://localhost:11434/v1"); + assert!(analyzer.api_key.is_none()); + assert_eq!(analyzer.model, "llama3"); + } + + #[tokio::test] + async fn test_openai_analyzer_empty_entries() { + let analyzer = OpenAiAnalyzer::new( + "http://localhost:11434/v1".into(), + None, + "llama3".into(), + ); + let summary = analyzer.summarize(&[]).await.unwrap(); + assert_eq!(summary.total_entries, 0); + } + + #[test] + fn test_log_summary_serialization() { + let summary = LogSummary { + source_id: "test".into(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 10, + summary_text: "All good".into(), + error_count: 0, + warning_count: 0, + key_events: vec!["Started".into()], + anomalies: vec![LogAnomaly { + description: "Test anomaly".into(), + severity: AnomalySeverity::Medium, + sample_line: "WARN: something".into(), + }], + }; + let json = serde_json::to_string(&summary).unwrap(); + let deserialized: LogSummary = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.total_entries, 10); + assert_eq!(deserialized.anomalies[0].severity, AnomalySeverity::Medium); + } +} diff --git a/src/sniff/config.rs b/src/sniff/config.rs new file mode 100644 index 0000000..0fa0294 --- /dev/null +++ b/src/sniff/config.rs @@ -0,0 +1,311 @@ +//! Sniff configuration loaded from environment variables and CLI args + +use std::env; +use std::path::PathBuf; + +/// AI provider selection +#[derive(Debug, Clone, PartialEq)] +pub enum AiProvider { + /// OpenAI-compatible API (works with OpenAI, Ollama, vLLM, etc.) + OpenAi, + /// Local inference via Candle (requires `ml` feature) + Candle, +} + +impl AiProvider { + pub fn from_str(s: &str) -> Self { + match s.to_lowercase().as_str() { + "candle" => AiProvider::Candle, + // "ollama" uses the same OpenAI-compatible API client + "openai" | "ollama" => AiProvider::OpenAi, + _ => AiProvider::OpenAi, + } + } +} + +/// Configuration for the `stackdog sniff` command +#[derive(Debug, Clone)] +pub struct SniffConfig { + /// Run once then exit (vs continuous daemon mode) + pub once: bool, + /// Enable consume mode: archive + purge originals + pub consume: bool, + /// Output directory for archived/consumed logs + pub output_dir: PathBuf, + /// Additional log source paths (user-configured) + pub extra_sources: Vec, + /// Poll interval in seconds + pub interval_secs: u64, + /// AI provider to use for summarization + pub ai_provider: AiProvider, + /// AI API URL (for OpenAI-compatible providers) + pub ai_api_url: String, + /// AI API key (optional for local providers like Ollama) + pub ai_api_key: Option, + /// AI model name + pub ai_model: String, + /// Database URL + pub database_url: String, + /// Slack webhook URL for alert notifications + pub slack_webhook: Option, + /// Generic webhook URL for alert notifications + pub webhook_url: Option, +} + +impl SniffConfig { + /// Build config from environment variables, overridden by CLI args + pub fn from_env_and_args( + once: bool, + consume: bool, + output: &str, + sources: Option<&str>, + interval: u64, + ai_provider_arg: Option<&str>, + ai_model_arg: Option<&str>, + ai_api_url_arg: Option<&str>, + slack_webhook_arg: Option<&str>, + ) -> Self { + let env_sources = env::var("STACKDOG_LOG_SOURCES").unwrap_or_default(); + let mut extra_sources: Vec = env_sources + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + + if let Some(cli_sources) = sources { + for s in cli_sources.split(',') { + let trimmed = s.trim().to_string(); + if !trimmed.is_empty() && !extra_sources.contains(&trimmed) { + extra_sources.push(trimmed); + } + } + } + + let ai_provider_str = ai_provider_arg + .map(|s| s.to_string()) + .unwrap_or_else(|| env::var("STACKDOG_AI_PROVIDER").unwrap_or_else(|_| "openai".into())); + + let output_dir = if output != "./stackdog-logs/" { + PathBuf::from(output) + } else { + PathBuf::from( + env::var("STACKDOG_SNIFF_OUTPUT_DIR") + .unwrap_or_else(|_| output.to_string()), + ) + }; + + let interval_secs = if interval != 30 { + interval + } else { + env::var("STACKDOG_SNIFF_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(interval) + }; + + Self { + once, + consume, + output_dir, + extra_sources, + interval_secs, + ai_provider: AiProvider::from_str(&ai_provider_str), + ai_api_url: ai_api_url_arg + .map(|s| s.to_string()) + .or_else(|| env::var("STACKDOG_AI_API_URL").ok()) + .unwrap_or_else(|| "http://localhost:11434/v1".into()), + ai_api_key: env::var("STACKDOG_AI_API_KEY").ok(), + ai_model: ai_model_arg + .map(|s| s.to_string()) + .or_else(|| env::var("STACKDOG_AI_MODEL").ok()) + .unwrap_or_else(|| "llama3".into()), + database_url: env::var("DATABASE_URL") + .unwrap_or_else(|_| "./stackdog.db".into()), + slack_webhook: slack_webhook_arg + .map(|s| s.to_string()) + .or_else(|| env::var("STACKDOG_SLACK_WEBHOOK_URL").ok()), + webhook_url: env::var("STACKDOG_WEBHOOK_URL").ok(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + // Serialize env-mutating tests to avoid cross-contamination + static ENV_MUTEX: Mutex<()> = Mutex::new(()); + + fn clear_sniff_env() { + env::remove_var("STACKDOG_LOG_SOURCES"); + env::remove_var("STACKDOG_AI_PROVIDER"); + env::remove_var("STACKDOG_AI_API_URL"); + env::remove_var("STACKDOG_AI_API_KEY"); + env::remove_var("STACKDOG_AI_MODEL"); + env::remove_var("STACKDOG_SNIFF_OUTPUT_DIR"); + env::remove_var("STACKDOG_SNIFF_INTERVAL"); + env::remove_var("STACKDOG_SLACK_WEBHOOK_URL"); + env::remove_var("STACKDOG_WEBHOOK_URL"); + } + + #[test] + fn test_ai_provider_from_str() { + assert_eq!(AiProvider::from_str("openai"), AiProvider::OpenAi); + assert_eq!(AiProvider::from_str("OpenAI"), AiProvider::OpenAi); + assert_eq!(AiProvider::from_str("candle"), AiProvider::Candle); + assert_eq!(AiProvider::from_str("Candle"), AiProvider::Candle); + assert_eq!(AiProvider::from_str("unknown"), AiProvider::OpenAi); + } + + #[test] + fn test_sniff_config_defaults() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args(false, false, "./stackdog-logs/", None, 30, None, None, None, None); + assert!(!config.once); + assert!(!config.consume); + assert_eq!(config.output_dir, PathBuf::from("./stackdog-logs/")); + assert!(config.extra_sources.is_empty()); + assert_eq!(config.interval_secs, 30); + assert_eq!(config.ai_provider, AiProvider::OpenAi); + assert_eq!(config.ai_api_url, "http://localhost:11434/v1"); + assert!(config.ai_api_key.is_none()); + assert_eq!(config.ai_model, "llama3"); + } + + #[test] + fn test_sniff_config_cli_overrides() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args( + true, true, "/tmp/output/", Some("/var/log/app.log"), 60, Some("candle"), None, None, None, + ); + + assert!(config.once); + assert!(config.consume); + assert_eq!(config.output_dir, PathBuf::from("/tmp/output/")); + assert_eq!(config.extra_sources, vec!["/var/log/app.log"]); + assert_eq!(config.interval_secs, 60); + assert_eq!(config.ai_provider, AiProvider::Candle); + } + + #[test] + fn test_sniff_config_env_sources_merged_with_cli() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_LOG_SOURCES", "/var/log/syslog,/var/log/auth.log"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", Some("/var/log/app.log,/var/log/syslog"), 30, None, None, None, None, + ); + + assert!(config.extra_sources.contains(&"/var/log/syslog".to_string())); + assert!(config.extra_sources.contains(&"/var/log/auth.log".to_string())); + assert!(config.extra_sources.contains(&"/var/log/app.log".to_string())); + assert_eq!(config.extra_sources.len(), 3); + + clear_sniff_env(); + } + + #[test] + fn test_sniff_config_env_overrides_defaults() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_AI_API_URL", "https://api.openai.com/v1"); + env::set_var("STACKDOG_AI_API_KEY", "sk-test123"); + env::set_var("STACKDOG_AI_MODEL", "gpt-4o-mini"); + env::set_var("STACKDOG_SNIFF_INTERVAL", "45"); + env::set_var("STACKDOG_SNIFF_OUTPUT_DIR", "/data/logs/"); + + let config = SniffConfig::from_env_and_args(false, false, "./stackdog-logs/", None, 30, None, None, None, None); + assert_eq!(config.ai_api_url, "https://api.openai.com/v1"); + assert_eq!(config.ai_api_key, Some("sk-test123".into())); + assert_eq!(config.ai_model, "gpt-4o-mini"); + assert_eq!(config.interval_secs, 45); + assert_eq!(config.output_dir, PathBuf::from("/data/logs/")); + + clear_sniff_env(); + } + + #[test] + fn test_ollama_provider_alias() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + Some("ollama"), Some("qwen2.5-coder:latest"), None, None, + ); + // "ollama" maps to OpenAi internally (same API protocol) + assert_eq!(config.ai_provider, AiProvider::OpenAi); + assert_eq!(config.ai_model, "qwen2.5-coder:latest"); + assert_eq!(config.ai_api_url, "http://localhost:11434/v1"); + + clear_sniff_env(); + } + + #[test] + fn test_cli_args_override_env_vars() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_AI_MODEL", "gpt-4o-mini"); + env::set_var("STACKDOG_AI_API_URL", "https://api.openai.com/v1"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, Some("llama3"), Some("http://localhost:11434/v1"), None, + ); + // CLI args take priority over env vars + assert_eq!(config.ai_model, "llama3"); + assert_eq!(config.ai_api_url, "http://localhost:11434/v1"); + + clear_sniff_env(); + } + + #[test] + fn test_slack_webhook_from_cli() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, None, None, Some("https://hooks.slack.com/services/T/B/xxx"), + ); + assert_eq!(config.slack_webhook.as_deref(), Some("https://hooks.slack.com/services/T/B/xxx")); + + clear_sniff_env(); + } + + #[test] + fn test_slack_webhook_from_env() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T/B/env"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, None, None, None, + ); + assert_eq!(config.slack_webhook.as_deref(), Some("https://hooks.slack.com/services/T/B/env")); + + clear_sniff_env(); + } + + #[test] + fn test_slack_webhook_cli_overrides_env() { + let _lock = ENV_MUTEX.lock().unwrap(); + clear_sniff_env(); + env::set_var("STACKDOG_SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T/B/env"); + + let config = SniffConfig::from_env_and_args( + false, false, "./stackdog-logs/", None, 30, + None, None, None, Some("https://hooks.slack.com/services/T/B/cli"), + ); + assert_eq!(config.slack_webhook.as_deref(), Some("https://hooks.slack.com/services/T/B/cli")); + + clear_sniff_env(); + } +} diff --git a/src/sniff/consumer.rs b/src/sniff/consumer.rs new file mode 100644 index 0000000..b594a63 --- /dev/null +++ b/src/sniff/consumer.rs @@ -0,0 +1,352 @@ +//! Log consumer: compress, deduplicate, and purge original logs +//! +//! When `--consume` is enabled, logs are archived to zstd-compressed files, +//! deduplicated, and then originals are purged to free disk space. + +use anyhow::{Result, Context}; +use chrono::Utc; +use std::collections::HashSet; +use std::collections::hash_map::DefaultHasher; +use std::fs::{self, File, OpenOptions}; +use std::hash::{Hash, Hasher}; +use std::io::{Write, BufWriter}; +use std::path::{Path, PathBuf}; + +use crate::sniff::reader::LogEntry; +use crate::sniff::discovery::LogSourceType; + +/// Result of a consume operation +#[derive(Debug, Clone, Default)] +pub struct ConsumeResult { + pub entries_archived: usize, + pub duplicates_skipped: usize, + pub bytes_freed: u64, + pub compressed_size: u64, +} + +/// Consumes log entries: deduplicates, compresses to zstd, and purges originals +pub struct LogConsumer { + output_dir: PathBuf, + seen_hashes: HashSet, + max_seen_hashes: usize, +} + +impl LogConsumer { + pub fn new(output_dir: PathBuf) -> Result { + fs::create_dir_all(&output_dir) + .with_context(|| format!("Failed to create output directory: {}", output_dir.display()))?; + + Ok(Self { + output_dir, + seen_hashes: HashSet::new(), + max_seen_hashes: 100_000, + }) + } + + /// Hash a log line for deduplication + fn hash_line(line: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + line.hash(&mut hasher); + hasher.finish() + } + + /// Deduplicate entries, returning only unique ones + pub fn deduplicate<'a>(&mut self, entries: &'a [LogEntry]) -> Vec<&'a LogEntry> { + // Evict oldest hashes if at capacity + if self.seen_hashes.len() > self.max_seen_hashes { + self.seen_hashes.clear(); + } + + let seen = &mut self.seen_hashes; + entries.iter().filter(|entry| { + let hash = Self::hash_line(&entry.line); + seen.insert(hash) + }).collect() + } + + /// Write entries to a zstd-compressed file + pub fn write_compressed(&self, entries: &[&LogEntry], source_name: &str) -> Result<(PathBuf, u64)> { + let timestamp = Utc::now().format("%Y%m%d_%H%M%S"); + let safe_name = source_name.replace(['/', '\\', ':', ' '], "_"); + let filename = format!("{}_{}.log.zst", safe_name, timestamp); + let path = self.output_dir.join(&filename); + + let file = File::create(&path) + .with_context(|| format!("Failed to create archive file: {}", path.display()))?; + + let encoder = zstd::Encoder::new(file, 3) + .context("Failed to create zstd encoder")?; + let mut writer = BufWriter::new(encoder); + + for entry in entries { + writeln!(writer, "{}\t{}", entry.timestamp.to_rfc3339(), entry.line)?; + } + + let encoder = writer.into_inner() + .map_err(|e| anyhow::anyhow!("Buffer flush error: {}", e))?; + encoder.finish() + .context("Failed to finish zstd encoding")?; + + let compressed_size = fs::metadata(&path)?.len(); + Ok((path, compressed_size)) + } + + /// Purge a file-based log source by truncating it + pub fn purge_file(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + + let original_size = fs::metadata(path)?.len(); + + // Truncate the file (preserves the fd for syslog daemons) + OpenOptions::new() + .write(true) + .truncate(true) + .open(path) + .with_context(|| format!("Failed to truncate log file: {}", path.display()))?; + + Ok(original_size) + } + + /// Purge Docker container logs by truncating the JSON log file + pub async fn purge_docker_logs(container_id: &str) -> Result { + // Docker stores logs at /var/lib/docker/containers//-json.log + let log_path = format!("/var/lib/docker/containers/{}/{}-json.log", container_id, container_id); + let path = Path::new(&log_path); + + if path.exists() { + Self::purge_file(path) + } else { + log::info!("Docker log file not found for container {}, skipping purge", container_id); + Ok(0) + } + } + + /// Full consume pipeline: deduplicate → compress → purge + pub async fn consume( + &mut self, + entries: &[LogEntry], + source_name: &str, + source_type: &LogSourceType, + source_path: &str, + ) -> Result { + if entries.is_empty() { + return Ok(ConsumeResult::default()); + } + + let total = entries.len(); + let unique_entries = self.deduplicate(entries); + let duplicates_skipped = total - unique_entries.len(); + + let (_, compressed_size) = self.write_compressed(&unique_entries, source_name)?; + + let bytes_freed = match source_type { + LogSourceType::DockerContainer => { + Self::purge_docker_logs(source_path).await? + } + LogSourceType::SystemLog | LogSourceType::CustomFile => { + let path = Path::new(source_path); + Self::purge_file(path)? + } + }; + + Ok(ConsumeResult { + entries_archived: unique_entries.len(), + duplicates_skipped, + bytes_freed, + compressed_size, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use std::collections::HashMap; + use std::io::Read; + + fn make_entry(line: &str) -> LogEntry { + LogEntry { + source_id: "test".into(), + timestamp: Utc::now(), + line: line.to_string(), + metadata: HashMap::new(), + } + } + + fn make_entries(lines: &[&str]) -> Vec { + lines.iter().map(|l| make_entry(l)).collect() + } + + #[test] + fn test_hash_line_deterministic() { + let h1 = LogConsumer::hash_line("hello world"); + let h2 = LogConsumer::hash_line("hello world"); + assert_eq!(h1, h2); + } + + #[test] + fn test_hash_line_different_for_different_inputs() { + let h1 = LogConsumer::hash_line("hello"); + let h2 = LogConsumer::hash_line("world"); + assert_ne!(h1, h2); + } + + #[test] + fn test_deduplicate_removes_duplicates() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["line A", "line B", "line A", "line C", "line B"]); + let unique = consumer.deduplicate(&entries); + assert_eq!(unique.len(), 3); + } + + #[test] + fn test_deduplicate_all_unique() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["line 1", "line 2", "line 3"]); + let unique = consumer.deduplicate(&entries); + assert_eq!(unique.len(), 3); + } + + #[test] + fn test_deduplicate_all_same() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["same", "same", "same"]); + let unique = consumer.deduplicate(&entries); + assert_eq!(unique.len(), 1); + } + + #[test] + fn test_write_compressed_creates_file() { + let dir = tempfile::tempdir().unwrap(); + let consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["line 1", "line 2"]); + let refs: Vec<&LogEntry> = entries.iter().collect(); + let (path, size) = consumer.write_compressed(&refs, "test-source").unwrap(); + + assert!(path.exists()); + assert!(size > 0); + assert!(path.to_string_lossy().ends_with(".log.zst")); + } + + #[test] + fn test_write_compressed_is_valid_zstd() { + let dir = tempfile::tempdir().unwrap(); + let consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let entries = make_entries(&["test line 1", "test line 2"]); + let refs: Vec<&LogEntry> = entries.iter().collect(); + let (path, _) = consumer.write_compressed(&refs, "zstd-test").unwrap(); + + // Decompress and verify + let file = File::open(&path).unwrap(); + let mut decoder = zstd::Decoder::new(file).unwrap(); + let mut content = String::new(); + decoder.read_to_string(&mut content).unwrap(); + + assert!(content.contains("test line 1")); + assert!(content.contains("test line 2")); + } + + #[test] + fn test_purge_file_truncates() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("to_purge.log"); + { + let mut f = File::create(&path).unwrap(); + write!(f, "lots of log data here that takes up space").unwrap(); + } + + let original_size = fs::metadata(&path).unwrap().len(); + assert!(original_size > 0); + + let freed = LogConsumer::purge_file(&path).unwrap(); + assert_eq!(freed, original_size); + + let new_size = fs::metadata(&path).unwrap().len(); + assert_eq!(new_size, 0); + } + + #[test] + fn test_purge_file_nonexistent() { + let freed = LogConsumer::purge_file(Path::new("/nonexistent/file.log")).unwrap(); + assert_eq!(freed, 0); + } + + #[tokio::test] + async fn test_consume_full_pipeline() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("app.log"); + { + let mut f = File::create(&log_path).unwrap(); + writeln!(f, "line 1").unwrap(); + writeln!(f, "line 2").unwrap(); + writeln!(f, "line 1").unwrap(); // duplicate + } + + let output_dir = dir.path().join("output"); + let mut consumer = LogConsumer::new(output_dir.clone()).unwrap(); + + let entries = make_entries(&["line 1", "line 2", "line 1"]); + let log_path_str = log_path.to_string_lossy().to_string(); + + let result = consumer.consume( + &entries, + "app", + &LogSourceType::CustomFile, + &log_path_str, + ).await.unwrap(); + + assert_eq!(result.entries_archived, 2); // deduplicated + assert_eq!(result.duplicates_skipped, 1); + assert!(result.compressed_size > 0); + assert!(result.bytes_freed > 0); + + // Original file should be truncated + let size = fs::metadata(&log_path).unwrap().len(); + assert_eq!(size, 0); + } + + #[tokio::test] + async fn test_consume_empty_entries() { + let dir = tempfile::tempdir().unwrap(); + let mut consumer = LogConsumer::new(dir.path().to_path_buf()).unwrap(); + + let result = consumer.consume( + &[], + "empty", + &LogSourceType::SystemLog, + "/var/log/test", + ).await.unwrap(); + + assert_eq!(result.entries_archived, 0); + assert_eq!(result.duplicates_skipped, 0); + } + + #[test] + fn test_consumer_creates_output_dir() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("a/b/c"); + assert!(!nested.exists()); + + let consumer = LogConsumer::new(nested.clone()); + assert!(consumer.is_ok()); + assert!(nested.exists()); + } + + #[test] + fn test_consume_result_default() { + let result = ConsumeResult::default(); + assert_eq!(result.entries_archived, 0); + assert_eq!(result.bytes_freed, 0); + } +} diff --git a/src/sniff/discovery.rs b/src/sniff/discovery.rs new file mode 100644 index 0000000..c8acf92 --- /dev/null +++ b/src/sniff/discovery.rs @@ -0,0 +1,267 @@ +//! Log source discovery +//! +//! Scans for log sources across Docker containers, system log files, +//! and user-configured custom paths. + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +/// Type of log source +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum LogSourceType { + DockerContainer, + SystemLog, + CustomFile, +} + +impl std::fmt::Display for LogSourceType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LogSourceType::DockerContainer => write!(f, "DockerContainer"), + LogSourceType::SystemLog => write!(f, "SystemLog"), + LogSourceType::CustomFile => write!(f, "CustomFile"), + } + } +} + +impl LogSourceType { + pub fn from_str(s: &str) -> Self { + match s { + "DockerContainer" => LogSourceType::DockerContainer, + "SystemLog" => LogSourceType::SystemLog, + _ => LogSourceType::CustomFile, + } + } +} + +/// A discovered log source +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogSource { + pub id: String, + pub source_type: LogSourceType, + /// File path (for system/custom) or container ID (for Docker) + pub path_or_id: String, + pub name: String, + pub discovered_at: DateTime, + /// Byte offset for incremental reads (files only) + pub last_read_position: u64, +} + +impl LogSource { + pub fn new(source_type: LogSourceType, path_or_id: String, name: String) -> Self { + Self { + id: uuid::Uuid::new_v4().to_string(), + source_type, + path_or_id, + name, + discovered_at: Utc::now(), + last_read_position: 0, + } + } +} + +/// Well-known system log paths to probe +const SYSTEM_LOG_PATHS: &[&str] = &[ + "/var/log/syslog", + "/var/log/messages", + "/var/log/auth.log", + "/var/log/kern.log", + "/var/log/daemon.log", + "/var/log/secure", +]; + +/// Discover system log files that exist and are readable +pub fn discover_system_logs() -> Vec { + log::debug!("Probing {} system log paths", SYSTEM_LOG_PATHS.len()); + let sources: Vec = SYSTEM_LOG_PATHS + .iter() + .filter(|path| { + let exists = Path::new(path).exists(); + log::trace!("System log {} — exists: {}", path, exists); + exists + }) + .map(|path| { + let name = Path::new(path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + .to_string(); + LogSource::new(LogSourceType::SystemLog, path.to_string(), name) + }) + .collect(); + log::debug!("Discovered {} system log sources", sources.len()); + sources +} + +/// Register user-configured custom log file paths +pub fn discover_custom_sources(paths: &[String]) -> Vec { + log::debug!("Checking {} custom source paths", paths.len()); + paths + .iter() + .filter(|path| { + let exists = Path::new(path.as_str()).exists(); + if exists { + log::debug!("Custom source found: {}", path); + } else { + log::debug!("Custom source not found (skipped): {}", path); + } + exists + }) + .map(|path| { + let name = Path::new(path.as_str()) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("custom") + .to_string(); + LogSource::new(LogSourceType::CustomFile, path.clone(), name) + }) + .collect() +} + +/// Discover Docker container log sources +pub async fn discover_docker_sources() -> Result> { + use crate::docker::DockerClient; + + let client = match DockerClient::new().await { + Ok(c) => c, + Err(e) => { + log::warn!("Docker not available for log discovery: {}", e); + return Ok(Vec::new()); + } + }; + + let containers = client.list_containers(false).await?; + let sources = containers + .into_iter() + .map(|c| { + let name = format!("docker:{}", c.name); + LogSource::new(LogSourceType::DockerContainer, c.id, name) + }) + .collect(); + + Ok(sources) +} + +/// Run full discovery across all source types +pub async fn discover_all(extra_paths: &[String]) -> Result> { + let mut sources = Vec::new(); + + // System logs + let sys = discover_system_logs(); + log::debug!("System log discovery: {} sources", sys.len()); + sources.extend(sys); + + // Custom paths + let custom = discover_custom_sources(extra_paths); + log::debug!("Custom source discovery: {} sources", custom.len()); + sources.extend(custom); + + // Docker containers + match discover_docker_sources().await { + Ok(docker_sources) => { + log::debug!("Docker discovery: {} containers", docker_sources.len()); + sources.extend(docker_sources); + } + Err(e) => log::warn!("Docker discovery failed: {}", e), + } + + log::debug!("Total discovered sources: {}", sources.len()); + for s in &sources { + log::debug!(" [{:?}] {} — {}", s.source_type, s.name, s.path_or_id); + } + + Ok(sources) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_log_source_type_display() { + assert_eq!(LogSourceType::DockerContainer.to_string(), "DockerContainer"); + assert_eq!(LogSourceType::SystemLog.to_string(), "SystemLog"); + assert_eq!(LogSourceType::CustomFile.to_string(), "CustomFile"); + } + + #[test] + fn test_log_source_type_from_str() { + assert_eq!(LogSourceType::from_str("DockerContainer"), LogSourceType::DockerContainer); + assert_eq!(LogSourceType::from_str("SystemLog"), LogSourceType::SystemLog); + assert_eq!(LogSourceType::from_str("CustomFile"), LogSourceType::CustomFile); + assert_eq!(LogSourceType::from_str("anything"), LogSourceType::CustomFile); + } + + #[test] + fn test_log_source_new() { + let source = LogSource::new( + LogSourceType::SystemLog, + "/var/log/syslog".into(), + "syslog".into(), + ); + assert_eq!(source.source_type, LogSourceType::SystemLog); + assert_eq!(source.path_or_id, "/var/log/syslog"); + assert_eq!(source.name, "syslog"); + assert_eq!(source.last_read_position, 0); + assert!(!source.id.is_empty()); + } + + #[test] + fn test_discover_custom_sources_existing_file() { + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "test log line").unwrap(); + let path = tmp.path().to_string_lossy().to_string(); + + let sources = discover_custom_sources(&[path.clone()]); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].source_type, LogSourceType::CustomFile); + assert_eq!(sources[0].path_or_id, path); + } + + #[test] + fn test_discover_custom_sources_nonexistent_file() { + let sources = discover_custom_sources(&["/nonexistent/path/log.txt".into()]); + assert!(sources.is_empty()); + } + + #[test] + fn test_discover_custom_sources_mixed() { + let mut tmp = NamedTempFile::new().unwrap(); + writeln!(tmp, "log").unwrap(); + let existing = tmp.path().to_string_lossy().to_string(); + + let sources = discover_custom_sources(&[ + existing.clone(), + "/does/not/exist.log".into(), + ]); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].path_or_id, existing); + } + + #[test] + fn test_discover_system_logs_returns_only_existing() { + let sources = discover_system_logs(); + for source in &sources { + assert_eq!(source.source_type, LogSourceType::SystemLog); + assert!(Path::new(&source.path_or_id).exists()); + } + } + + #[test] + fn test_log_source_serialization() { + let source = LogSource::new( + LogSourceType::DockerContainer, + "abc123def456".into(), + "docker:myapp".into(), + ); + let json = serde_json::to_string(&source).unwrap(); + let deserialized: LogSource = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.source_type, LogSourceType::DockerContainer); + assert_eq!(deserialized.path_or_id, "abc123def456"); + assert_eq!(deserialized.name, "docker:myapp"); + } +} diff --git a/src/sniff/mod.rs b/src/sniff/mod.rs new file mode 100644 index 0000000..4372bd2 --- /dev/null +++ b/src/sniff/mod.rs @@ -0,0 +1,268 @@ +//! Log sniffing module +//! +//! Discovers, reads, analyzes, and optionally consumes logs from +//! Docker containers, system log files, and custom sources. + +pub mod config; +pub mod discovery; +pub mod reader; +pub mod analyzer; +pub mod consumer; +pub mod reporter; + +use anyhow::Result; +use crate::database::connection::{create_pool, init_database, DbPool}; +use crate::alerting::notifications::NotificationConfig; +use crate::sniff::config::SniffConfig; +use crate::sniff::discovery::LogSourceType; +use crate::sniff::reader::{LogReader, FileLogReader, DockerLogReader}; +use crate::sniff::analyzer::{LogAnalyzer, PatternAnalyzer}; +use crate::sniff::consumer::LogConsumer; +use crate::sniff::reporter::Reporter; +use crate::database::repositories::log_sources as log_sources_repo; + +/// Main orchestrator for the sniff command +pub struct SniffOrchestrator { + config: SniffConfig, + pool: DbPool, + reporter: Reporter, +} + +impl SniffOrchestrator { + pub fn new(config: SniffConfig) -> Result { + let pool = create_pool(&config.database_url)?; + init_database(&pool)?; + + let mut notification_config = NotificationConfig::default(); + if let Some(ref url) = config.slack_webhook { + notification_config = notification_config.with_slack_webhook(url.clone()); + } + if let Some(ref url) = config.webhook_url { + notification_config = notification_config.with_webhook_url(url.clone()); + } + let reporter = Reporter::new(notification_config); + + Ok(Self { config, pool, reporter }) + } + + /// Create the appropriate AI analyzer based on config + fn create_analyzer(&self) -> Box { + match self.config.ai_provider { + config::AiProvider::OpenAi => { + log::debug!( + "Creating OpenAI-compatible analyzer (model: {}, url: {})", + self.config.ai_model, self.config.ai_api_url + ); + Box::new(analyzer::OpenAiAnalyzer::new( + self.config.ai_api_url.clone(), + self.config.ai_api_key.clone(), + self.config.ai_model.clone(), + )) + } + config::AiProvider::Candle => { + log::info!("Using pattern analyzer (Candle backend not yet implemented)"); + Box::new(PatternAnalyzer::new()) + } + } + } + + /// Build readers for discovered sources, restoring saved positions from DB + fn build_readers(&self, sources: &[discovery::LogSource]) -> Vec> { + sources.iter().filter_map(|source| { + let saved = log_sources_repo::get_log_source_by_path(&self.pool, &source.path_or_id) + .ok() + .flatten(); + let offset = saved.map(|s| s.last_read_position).unwrap_or(0); + + match source.source_type { + LogSourceType::SystemLog | LogSourceType::CustomFile => { + Some(Box::new(FileLogReader::new( + source.id.clone(), + source.path_or_id.clone(), + offset, + )) as Box) + } + LogSourceType::DockerContainer => { + Some(Box::new(DockerLogReader::new( + source.id.clone(), + source.path_or_id.clone(), + )) as Box) + } + } + }).collect() + } + + /// Run a single sniff pass: discover → read → analyze → report → consume + pub async fn run_once(&self) -> Result { + let mut result = SniffPassResult::default(); + + // 1. Discover sources + log::debug!("Step 1: discovering log sources..."); + let sources = discovery::discover_all(&self.config.extra_sources).await?; + result.sources_found = sources.len(); + log::debug!("Discovered {} sources", sources.len()); + + // Register sources in DB + for source in &sources { + let _ = log_sources_repo::upsert_log_source(&self.pool, source); + } + + // 2. Build readers and analyzer + log::debug!("Step 2: building readers and analyzer..."); + let mut readers = self.build_readers(&sources); + let analyzer = self.create_analyzer(); + let mut consumer = if self.config.consume { + log::debug!("Consume mode enabled, output: {}", self.config.output_dir.display()); + Some(LogConsumer::new(self.config.output_dir.clone())?) + } else { + None + }; + + // 3. Process each source + let reader_count = readers.len(); + for (i, reader) in readers.iter_mut().enumerate() { + log::debug!("Step 3: reading source {}/{} ({})", i + 1, reader_count, reader.source_id()); + let entries = reader.read_new_entries().await?; + if entries.is_empty() { + log::debug!(" No new entries, skipping"); + continue; + } + + result.total_entries += entries.len(); + log::debug!(" Read {} entries", entries.len()); + + // 4. Analyze + log::debug!("Step 4: analyzing {} entries...", entries.len()); + let summary = analyzer.summarize(&entries).await?; + log::debug!( + " Analysis complete: {} errors, {} warnings, {} anomalies", + summary.error_count, summary.warning_count, summary.anomalies.len() + ); + + // 5. Report + log::debug!("Step 5: reporting results..."); + let report = self.reporter.report(&summary, Some(&self.pool))?; + result.anomalies_found += report.anomalies_reported; + + // 6. Consume (if enabled) + if let Some(ref mut cons) = consumer { + if i < sources.len() { + log::debug!("Step 6: consuming entries..."); + let source = &sources[i]; + let consume_result = cons.consume( + &entries, + &source.name, + &source.source_type, + &source.path_or_id, + ).await?; + result.bytes_freed += consume_result.bytes_freed; + result.entries_archived += consume_result.entries_archived; + log::debug!(" Consumed: {} archived, {} bytes freed", + consume_result.entries_archived, consume_result.bytes_freed); + } + } + + // 7. Update read position + log::debug!("Step 7: saving read position ({})", reader.position()); + let _ = log_sources_repo::update_read_position( + &self.pool, + reader.source_id(), + reader.position(), + ); + } + + Ok(result) + } + + /// Run the sniff loop (continuous or one-shot) + pub async fn run(&self) -> Result<()> { + log::info!("🔍 Sniff orchestrator started"); + + loop { + match self.run_once().await { + Ok(result) => { + log::info!( + "Sniff pass: {} sources, {} entries, {} anomalies, {} bytes freed", + result.sources_found, + result.total_entries, + result.anomalies_found, + result.bytes_freed, + ); + } + Err(e) => { + log::error!("Sniff pass failed: {}", e); + } + } + + if self.config.once { + log::info!("🏁 One-shot mode: exiting after single pass"); + break; + } + + tokio::time::sleep(tokio::time::Duration::from_secs(self.config.interval_secs)).await; + } + + Ok(()) + } +} + +/// Result of a single sniff pass +#[derive(Debug, Clone, Default)] +pub struct SniffPassResult { + pub sources_found: usize, + pub total_entries: usize, + pub anomalies_found: usize, + pub bytes_freed: u64, + pub entries_archived: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sniff_pass_result_default() { + let result = SniffPassResult::default(); + assert_eq!(result.sources_found, 0); + assert_eq!(result.total_entries, 0); + assert_eq!(result.anomalies_found, 0); + assert_eq!(result.bytes_freed, 0); + } + + #[test] + fn test_orchestrator_creates_with_memory_db() { + let mut config = SniffConfig::from_env_and_args( + true, false, "./stackdog-logs/", None, 30, None, None, None, None, + ); + config.database_url = ":memory:".into(); + + let orchestrator = SniffOrchestrator::new(config); + assert!(orchestrator.is_ok()); + } + + #[tokio::test] + async fn test_orchestrator_run_once_with_file() { + use std::io::Write; + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("test.log"); + { + let mut f = std::fs::File::create(&log_path).unwrap(); + writeln!(f, "INFO: service started").unwrap(); + writeln!(f, "ERROR: connection failed").unwrap(); + writeln!(f, "WARN: retry in 5s").unwrap(); + } + + let mut config = SniffConfig::from_env_and_args( + true, false, "./stackdog-logs/", + Some(&log_path.to_string_lossy()), + 30, Some("candle"), None, None, None, + ); + config.database_url = ":memory:".into(); + + let orchestrator = SniffOrchestrator::new(config).unwrap(); + let result = orchestrator.run_once().await.unwrap(); + + assert!(result.sources_found >= 1); + assert!(result.total_entries >= 3); + } +} diff --git a/src/sniff/reader.rs b/src/sniff/reader.rs new file mode 100644 index 0000000..f97cabf --- /dev/null +++ b/src/sniff/reader.rs @@ -0,0 +1,423 @@ +//! Log readers for different source types +//! +//! Implements the `LogReader` trait for file-based logs, Docker container logs, +//! and systemd journal (Linux only). + +use anyhow::Result; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::fs::File; +use std::path::Path; + +/// A single log entry from any source +#[derive(Debug, Clone)] +pub struct LogEntry { + pub source_id: String, + pub timestamp: DateTime, + pub line: String, + pub metadata: HashMap, +} + +/// Trait for reading log entries from a source +#[async_trait] +pub trait LogReader: Send + Sync { + /// Read new entries since the last read position + async fn read_new_entries(&mut self) -> Result>; + /// Return the source identifier + fn source_id(&self) -> &str; + /// Return current read position (bytes for files, opaque for others) + fn position(&self) -> u64; +} + +/// Reads log entries from a regular file, tracking byte offset +pub struct FileLogReader { + source_id: String, + path: String, + offset: u64, +} + +impl FileLogReader { + pub fn new(source_id: String, path: String, start_offset: u64) -> Self { + Self { + source_id, + path, + offset: start_offset, + } + } + + fn read_lines_from_offset(&mut self) -> Result> { + let path = Path::new(&self.path); + if !path.exists() { + log::debug!("Log file does not exist: {}", self.path); + return Ok(Vec::new()); + } + + let file = File::open(path)?; + let file_len = file.metadata()?.len(); + log::debug!("Reading {} (size: {} bytes, offset: {})", self.path, file_len, self.offset); + + // Handle file truncation (log rotation) + if self.offset > file_len { + log::debug!("File truncated (rotation?), resetting offset from {} to 0", self.offset); + self.offset = 0; + } + + let mut reader = BufReader::new(file); + reader.seek(SeekFrom::Start(self.offset))?; + + let mut entries = Vec::new(); + let mut line = String::new(); + + while reader.read_line(&mut line)? > 0 { + let trimmed = line.trim_end().to_string(); + if !trimmed.is_empty() { + entries.push(LogEntry { + source_id: self.source_id.clone(), + timestamp: Utc::now(), + line: trimmed, + metadata: HashMap::from([ + ("source_path".into(), self.path.clone()), + ]), + }); + } + line.clear(); + } + + self.offset = reader.stream_position()?; + log::debug!("Read {} entries from {}, new offset: {}", entries.len(), self.path, self.offset); + Ok(entries) + } +} + +#[async_trait] +impl LogReader for FileLogReader { + async fn read_new_entries(&mut self) -> Result> { + self.read_lines_from_offset() + } + + fn source_id(&self) -> &str { + &self.source_id + } + + fn position(&self) -> u64 { + self.offset + } +} + +/// Reads logs from a Docker container via the bollard API +pub struct DockerLogReader { + source_id: String, + container_id: String, + last_timestamp: Option, +} + +impl DockerLogReader { + pub fn new(source_id: String, container_id: String) -> Self { + Self { + source_id, + container_id, + last_timestamp: None, + } + } +} + +#[async_trait] +impl LogReader for DockerLogReader { + async fn read_new_entries(&mut self) -> Result> { + use bollard::Docker; + use bollard::container::LogsOptions; + use futures_util::stream::StreamExt; + + let docker = match Docker::connect_with_local_defaults() { + Ok(d) => d, + Err(e) => { + log::warn!("Docker not available: {}", e); + return Ok(Vec::new()); + } + }; + + let options = LogsOptions:: { + stdout: true, + stderr: true, + since: self.last_timestamp.unwrap_or(0), + timestamps: true, + tail: if self.last_timestamp.is_none() { "100".to_string() } else { "all".to_string() }, + ..Default::default() + }; + + let mut stream = docker.logs(&self.container_id, Some(options)); + let mut entries = Vec::new(); + + while let Some(result) = stream.next().await { + match result { + Ok(output) => { + let line = output.to_string(); + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() { + entries.push(LogEntry { + source_id: self.source_id.clone(), + timestamp: Utc::now(), + line: trimmed, + metadata: HashMap::from([ + ("container_id".into(), self.container_id.clone()), + ]), + }); + } + } + Err(e) => { + log::warn!("Error reading Docker logs for {}: {}", self.container_id, e); + break; + } + } + } + + self.last_timestamp = Some(Utc::now().timestamp()); + Ok(entries) + } + + fn source_id(&self) -> &str { + &self.source_id + } + + fn position(&self) -> u64 { + self.last_timestamp.unwrap_or(0) as u64 + } +} + +/// Reads logs from systemd journal (Linux only) +#[cfg(target_os = "linux")] +pub struct JournaldReader { + source_id: String, + cursor: Option, +} + +#[cfg(target_os = "linux")] +impl JournaldReader { + pub fn new(source_id: String) -> Self { + Self { + source_id, + cursor: None, + } + } +} + +#[cfg(target_os = "linux")] +#[async_trait] +impl LogReader for JournaldReader { + async fn read_new_entries(&mut self) -> Result> { + use tokio::process::Command; + + let mut cmd = Command::new("journalctl"); + cmd.arg("--no-pager") + .arg("-o").arg("short-iso") + .arg("-n").arg("200"); + + if let Some(ref cursor) = self.cursor { + cmd.arg("--after-cursor").arg(cursor); + } + + cmd.arg("--show-cursor"); + + let output = cmd.output().await?; + let stdout = String::from_utf8_lossy(&output.stdout); + let mut entries = Vec::new(); + + for line in stdout.lines() { + if line.starts_with("-- cursor:") { + self.cursor = line.strip_prefix("-- cursor: ").map(|s| s.to_string()); + continue; + } + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() { + entries.push(LogEntry { + source_id: self.source_id.clone(), + timestamp: Utc::now(), + line: trimmed, + metadata: HashMap::from([ + ("source".into(), "journald".into()), + ]), + }); + } + } + + Ok(entries) + } + + fn source_id(&self) -> &str { + &self.source_id + } + + fn position(&self) -> u64 { + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_log_entry_creation() { + let entry = LogEntry { + source_id: "test-source".into(), + timestamp: Utc::now(), + line: "Error: something went wrong".into(), + metadata: HashMap::from([("key".into(), "value".into())]), + }; + assert_eq!(entry.source_id, "test-source"); + assert!(entry.line.contains("Error")); + assert_eq!(entry.metadata.get("key"), Some(&"value".to_string())); + } + + #[test] + fn test_file_log_reader_new() { + let reader = FileLogReader::new("src-1".into(), "/tmp/test.log".into(), 0); + assert_eq!(reader.source_id(), "src-1"); + assert_eq!(reader.position(), 0); + } + + #[tokio::test] + async fn test_file_log_reader_reads_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "line 1").unwrap(); + writeln!(f, "line 2").unwrap(); + writeln!(f, "line 3").unwrap(); + } + + let mut reader = FileLogReader::new( + "test".into(), + path.to_string_lossy().to_string(), + 0, + ); + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].line, "line 1"); + assert_eq!(entries[1].line, "line 2"); + assert_eq!(entries[2].line, "line 3"); + } + + #[tokio::test] + async fn test_file_log_reader_incremental_reads() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("incremental.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "line A").unwrap(); + writeln!(f, "line B").unwrap(); + } + + let path_str = path.to_string_lossy().to_string(); + let mut reader = FileLogReader::new("inc".into(), path_str, 0); + + // First read + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 2); + + // No new lines → empty + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 0); + + // Append new lines + { + let mut f = std::fs::OpenOptions::new().append(true).open(&path).unwrap(); + writeln!(f, "line C").unwrap(); + } + + // Should only get the new line + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].line, "line C"); + } + + #[tokio::test] + async fn test_file_log_reader_handles_truncation() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("rotating.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "original long line with lots of content here").unwrap(); + } + + let path_str = path.to_string_lossy().to_string(); + let mut reader = FileLogReader::new("rot".into(), path_str, 0); + + // Read past original content + reader.read_new_entries().await.unwrap(); + let saved_pos = reader.position(); + assert!(saved_pos > 0); + + // Simulate log rotation: truncate and write shorter content + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "new").unwrap(); + } + + // Should detect truncation and read from beginning + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].line, "new"); + } + + #[tokio::test] + async fn test_file_log_reader_nonexistent_file() { + let mut reader = FileLogReader::new("missing".into(), "/nonexistent/file.log".into(), 0); + let entries = reader.read_new_entries().await.unwrap(); + assert!(entries.is_empty()); + } + + #[tokio::test] + async fn test_file_log_reader_skips_empty_lines() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("empty_lines.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "line 1").unwrap(); + writeln!(f).unwrap(); // empty line + writeln!(f, "line 3").unwrap(); + } + + let mut reader = FileLogReader::new( + "empty".into(), + path.to_string_lossy().to_string(), + 0, + ); + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].line, "line 1"); + assert_eq!(entries[1].line, "line 3"); + } + + #[tokio::test] + async fn test_file_log_reader_metadata_contains_path() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("meta.log"); + { + let mut f = File::create(&path).unwrap(); + writeln!(f, "test").unwrap(); + } + + let path_str = path.to_string_lossy().to_string(); + let mut reader = FileLogReader::new("meta".into(), path_str.clone(), 0); + let entries = reader.read_new_entries().await.unwrap(); + assert_eq!(entries[0].metadata.get("source_path"), Some(&path_str)); + } + + #[test] + fn test_docker_log_reader_new() { + let reader = DockerLogReader::new("d-1".into(), "abc123".into()); + assert_eq!(reader.source_id(), "d-1"); + assert_eq!(reader.position(), 0); + } + + #[test] + fn test_file_log_reader_with_start_offset() { + let reader = FileLogReader::new("off".into(), "/tmp/test.log".into(), 1024); + assert_eq!(reader.position(), 1024); + } +} diff --git a/src/sniff/reporter.rs b/src/sniff/reporter.rs new file mode 100644 index 0000000..bfc3b55 --- /dev/null +++ b/src/sniff/reporter.rs @@ -0,0 +1,209 @@ +//! Log analysis reporter +//! +//! Converts log summaries and anomalies into alerts, then dispatches +//! them via the existing notification channels. + +use anyhow::Result; +use crate::alerting::alert::{Alert, AlertSeverity, AlertType}; +use crate::alerting::notifications::{NotificationChannel, NotificationConfig, route_by_severity}; +use crate::sniff::analyzer::{LogSummary, LogAnomaly, AnomalySeverity}; +use crate::database::connection::DbPool; +use crate::database::repositories::log_sources; + +/// Reports log analysis results to alert channels and persists summaries +pub struct Reporter { + notification_config: NotificationConfig, +} + +impl Reporter { + pub fn new(notification_config: NotificationConfig) -> Self { + Self { notification_config } + } + + /// Map anomaly severity to alert severity + fn map_severity(anomaly_severity: &AnomalySeverity) -> AlertSeverity { + match anomaly_severity { + AnomalySeverity::Low => AlertSeverity::Low, + AnomalySeverity::Medium => AlertSeverity::Medium, + AnomalySeverity::High => AlertSeverity::High, + AnomalySeverity::Critical => AlertSeverity::Critical, + } + } + + /// Report a log summary: persist to DB and send anomaly alerts + pub fn report(&self, summary: &LogSummary, pool: Option<&DbPool>) -> Result { + let mut alerts_sent = 0; + + // Persist summary to database + if let Some(pool) = pool { + log::debug!("Persisting summary for source {} to database", summary.source_id); + let _ = log_sources::create_log_summary( + pool, + &summary.source_id, + &summary.summary_text, + &summary.period_start.to_rfc3339(), + &summary.period_end.to_rfc3339(), + summary.total_entries as i64, + summary.error_count as i64, + summary.warning_count as i64, + ); + } + + // Generate alerts for anomalies + for anomaly in &summary.anomalies { + let alert_severity = Self::map_severity(&anomaly.severity); + + log::debug!( + "Generating alert: severity={}, description={}", + anomaly.severity, anomaly.description + ); + + let alert = Alert::new( + AlertType::AnomalyDetected, + alert_severity, + format!( + "[Log Sniff] {} — Source: {} | Sample: {}", + anomaly.description, summary.source_id, anomaly.sample_line + ), + ); + + // Route to appropriate notification channels + let channels = route_by_severity(alert_severity); + log::debug!("Routing alert to {} notification channels", channels.len()); + for channel in &channels { + match channel.send(&alert, &self.notification_config) { + Ok(_) => alerts_sent += 1, + Err(e) => log::warn!("Failed to send notification: {}", e), + } + } + } + + // Log summary to console + log::info!( + "📊 Log Summary [{}]: {} entries, {} errors, {} warnings, {} anomalies", + summary.source_id, + summary.total_entries, + summary.error_count, + summary.warning_count, + summary.anomalies.len(), + ); + + Ok(ReportResult { + anomalies_reported: summary.anomalies.len(), + notifications_sent: alerts_sent, + summary_persisted: pool.is_some(), + }) + } +} + +/// Result of a report operation +#[derive(Debug, Clone, Default)] +pub struct ReportResult { + pub anomalies_reported: usize, + pub notifications_sent: usize, + pub summary_persisted: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use crate::database::connection::{create_pool, init_database}; + + fn make_summary(anomalies: Vec) -> LogSummary { + LogSummary { + source_id: "test-source".into(), + period_start: Utc::now(), + period_end: Utc::now(), + total_entries: 100, + summary_text: "Test summary".into(), + error_count: 5, + warning_count: 3, + key_events: vec!["Service restarted".into()], + anomalies, + } + } + + #[test] + fn test_map_severity() { + assert_eq!(Reporter::map_severity(&AnomalySeverity::Low), AlertSeverity::Low); + assert_eq!(Reporter::map_severity(&AnomalySeverity::Medium), AlertSeverity::Medium); + assert_eq!(Reporter::map_severity(&AnomalySeverity::High), AlertSeverity::High); + assert_eq!(Reporter::map_severity(&AnomalySeverity::Critical), AlertSeverity::Critical); + } + + #[test] + fn test_report_no_anomalies() { + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![]); + let result = reporter.report(&summary, None).unwrap(); + assert_eq!(result.anomalies_reported, 0); + assert_eq!(result.notifications_sent, 0); + assert!(!result.summary_persisted); + } + + #[test] + fn test_report_with_anomalies_sends_alerts() { + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![ + LogAnomaly { + description: "High error rate".into(), + severity: AnomalySeverity::High, + sample_line: "ERROR: connection failed".into(), + }, + ]); + + let result = reporter.report(&summary, None).unwrap(); + assert_eq!(result.anomalies_reported, 1); + // Console channel is always available, so at least 1 notification sent + assert!(result.notifications_sent >= 1); + } + + #[test] + fn test_report_persists_to_database() { + let pool = create_pool(":memory:").unwrap(); + init_database(&pool).unwrap(); + + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![]); + + let result = reporter.report(&summary, Some(&pool)).unwrap(); + assert!(result.summary_persisted); + + // Verify summary was stored + let summaries = log_sources::list_summaries_for_source(&pool, "test-source").unwrap(); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].total_entries, 100); + } + + #[test] + fn test_report_multiple_anomalies() { + let reporter = Reporter::new(NotificationConfig::default()); + let summary = make_summary(vec![ + LogAnomaly { + description: "Error spike".into(), + severity: AnomalySeverity::Critical, + sample_line: "FATAL: OOM".into(), + }, + LogAnomaly { + description: "Unusual pattern".into(), + severity: AnomalySeverity::Low, + sample_line: "DEBUG: retry".into(), + }, + ]); + + let result = reporter.report(&summary, None).unwrap(); + assert_eq!(result.anomalies_reported, 2); + assert!(result.notifications_sent >= 2); + } + + #[test] + fn test_reporter_new() { + let config = NotificationConfig::default(); + let reporter = Reporter::new(config); + // Just ensure it constructs without error + let summary = make_summary(vec![]); + let result = reporter.report(&summary, None); + assert!(result.is_ok()); + } +} diff --git a/tests/structure/mod_test.rs b/tests/structure/mod_test.rs index 4893d6a..ec4ea2b 100644 --- a/tests/structure/mod_test.rs +++ b/tests/structure/mod_test.rs @@ -5,64 +5,61 @@ #[test] fn test_collectors_module_imports() { - // Verify collectors module exists and can be imported - // This test will compile only if the module structure is correct - use crate::collectors; - - // Suppress unused import warning + use stackdog::collectors; let _ = std::marker::PhantomData::; } #[test] fn test_events_module_imports() { - use crate::events; + use stackdog::events; let _ = std::marker::PhantomData::; } #[test] fn test_rules_module_imports() { - use crate::rules; + use stackdog::rules; let _ = std::marker::PhantomData::; } #[test] fn test_ml_module_imports() { - use crate::ml; + use stackdog::ml; let _ = std::marker::PhantomData::; } +#[cfg(target_os = "linux")] #[test] fn test_firewall_module_imports() { - use crate::firewall; + use stackdog::firewall; let _ = std::marker::PhantomData::; } #[test] fn test_response_module_imports() { - use crate::response; + use stackdog::response; let _ = std::marker::PhantomData::; } #[test] fn test_correlator_module_imports() { - use crate::correlator; + use stackdog::correlator; let _ = std::marker::PhantomData::; } #[test] fn test_alerting_module_imports() { - use crate::alerting; + use stackdog::alerting; let _ = std::marker::PhantomData::; } #[test] fn test_baselines_module_imports() { - use crate::baselines; + use stackdog::baselines; let _ = std::marker::PhantomData::; } #[test] fn test_database_module_imports() { - use crate::database; + use stackdog::database; let _ = std::marker::PhantomData::; }