Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ dev_reports/
# Python artifacts
__pycache__/
*.py[cod]
.coverage
.pytest_cache/
htmlcov/

# Python virtual environments
.venv/
Expand Down
62 changes: 17 additions & 45 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,61 +39,33 @@ build:test:images:
config
docker push "$CI_REGISTRY_IMAGE/postgres-ai-configs:$PGAI_TAG"

# Build and push reporter
# Build and push reporter (TypeScript/Bun)
docker build \
--build-arg "VERSION=$PGAI_TAG" \
--build-arg "BUILD_TS=$BUILD_TS" \
-f reporter/Dockerfile \
-f cli/Dockerfile.reporter \
-t "$CI_REGISTRY_IMAGE/reporter:$PGAI_TAG" \
reporter
cli
docker push "$CI_REGISTRY_IMAGE/reporter:$PGAI_TAG"

# Build and push monitoring-flask-backend
# Build and push metrics-server (TypeScript/Bun, replaces monitoring-flask-backend)
docker build \
--build-arg "VERSION=$PGAI_TAG" \
--build-arg "BUILD_TS=$BUILD_TS" \
-f monitoring_flask_backend/Dockerfile \
-t "$CI_REGISTRY_IMAGE/monitoring-flask-backend:$PGAI_TAG" \
monitoring_flask_backend
docker push "$CI_REGISTRY_IMAGE/monitoring-flask-backend:$PGAI_TAG"
-f cli/Dockerfile.metrics-server \
-t "$CI_REGISTRY_IMAGE/metrics-server:$PGAI_TAG" \
cli
docker push "$CI_REGISTRY_IMAGE/metrics-server:$PGAI_TAG"

echo ""
echo "Images pushed to GitLab Container Registry:"
echo " $CI_REGISTRY_IMAGE/postgres-ai-configs:$PGAI_TAG"
echo " $CI_REGISTRY_IMAGE/reporter:$PGAI_TAG"
echo " $CI_REGISTRY_IMAGE/monitoring-flask-backend:$PGAI_TAG"
echo " $CI_REGISTRY_IMAGE/metrics-server:$PGAI_TAG"
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^feature\//'

reporter:tests:
stage: test
image: python:3.11-bullseye
variables:
GIT_STRATEGY: fetch
PIP_DISABLE_PIP_VERSION_CHECK: "1"
PIP_NO_CACHE_DIR: "1"
before_script:
- python --version
- pip install --upgrade pip
- apt-get update
- apt-get install -y --no-install-recommends postgresql postgresql-client && rm -rf /var/lib/apt/lists/*
- pip install -r reporter/requirements-dev.txt
script:
- chown -R postgres:postgres "$CI_PROJECT_DIR"
- su - postgres -c "cd \"$CI_PROJECT_DIR\" && python -m pytest --run-integration --cov=reporter --cov-report=term --cov-report=xml:coverage/reporter-coverage.xml tests/reporter"
# Fix ownership for artifact collection
- chown -R root:root "$CI_PROJECT_DIR/coverage" || true
coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+)%/'
artifacts:
when: always
paths:
- coverage/
expire_in: 7 days
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH == "main"'

cli:node:smoke:
stage: test
image: node:20-alpine
Expand Down Expand Up @@ -298,19 +270,19 @@ cli:npm:publish:
--platform "$PLATFORMS" \
--build-arg "VERSION=$VERSION" \
--build-arg "BUILD_TS=$BUILD_TS" \
-f reporter/Dockerfile \
-f cli/Dockerfile.reporter \
-t "postgresai/reporter:$VERSION" \
--push \
reporter
cli

docker buildx build \
--platform "$PLATFORMS" \
--build-arg "VERSION=$VERSION" \
--build-arg "BUILD_TS=$BUILD_TS" \
-f monitoring_flask_backend/Dockerfile \
-t "postgresai/monitoring-flask-backend:$VERSION" \
-f cli/Dockerfile.metrics-server \
-t "postgresai/metrics-server:$VERSION" \
--push \
monitoring_flask_backend
cli

docker buildx build \
--platform "$PLATFORMS" \
Expand All @@ -324,7 +296,7 @@ cli:npm:publish:
echo ""
echo "Published images:"
echo " postgresai/reporter:$VERSION"
echo " postgresai/monitoring-flask-backend:$VERSION"
echo " postgresai/metrics-server:$VERSION"
echo " postgresai/postgres-ai-configs:$VERSION"

docker:publish:images:
Expand Down Expand Up @@ -384,7 +356,7 @@ cli:node:e2e:dind:
echo "Pulling images from GitLab Container Registry..."
docker pull "$CI_REGISTRY_IMAGE/postgres-ai-configs:$PGAI_TAG"
docker pull "$CI_REGISTRY_IMAGE/reporter:$PGAI_TAG"
docker pull "$CI_REGISTRY_IMAGE/monitoring-flask-backend:$PGAI_TAG"
docker pull "$CI_REGISTRY_IMAGE/metrics-server:$PGAI_TAG"
echo "Images ready:"
docker images | grep "$CI_REGISTRY_IMAGE"
# Create .env file with registry and tag
Expand Down Expand Up @@ -425,7 +397,7 @@ cli:node:full:dind:
echo "Pulling images from GitLab Container Registry..."
docker pull "$CI_REGISTRY_IMAGE/postgres-ai-configs:$PGAI_TAG"
docker pull "$CI_REGISTRY_IMAGE/reporter:$PGAI_TAG"
docker pull "$CI_REGISTRY_IMAGE/monitoring-flask-backend:$PGAI_TAG"
docker pull "$CI_REGISTRY_IMAGE/metrics-server:$PGAI_TAG"
echo "Images ready:"
docker images | grep "$CI_REGISTRY_IMAGE"
# Create .env file with registry and tag
Expand Down
55 changes: 29 additions & 26 deletions .vscode/launch.example.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,39 +2,42 @@
"version": "0.2.0",
"configurations": [
{
"name": "Run Reporter (local)",
"type": "debugpy",
"name": "Run Reporter (TypeScript)",
"type": "node",
"request": "launch",
"module": "reporter.postgres_reports",
"env": {
"PYTHONPATH": "${workspaceFolder}"
},
"runtimeExecutable": "bun",
"runtimeArgs": ["run"],
"program": "${workspaceFolder}/cli/lib/reporter.ts",
"args": [
"--prometheus-url",
"http://127.0.0.1:59090",
"--postgres-sink-url",
"postgresql://pgwatch@127.0.0.1:55433/measurements",
"--no-upload",
"--output",
"-"
"http://127.0.0.1:59090"
],
"console": "integratedTerminal",
"justMyCode": true
"cwd": "${workspaceFolder}/cli",
"console": "integratedTerminal"
},
{
"name": "Attach (Flask in Docker: debugpy 5678)",
"type": "debugpy",
"request": "attach",
"connect": { "host": "127.0.0.1", "port": 5678 },
"pathMappings": [
{
"localRoot": "${workspaceFolder}/monitoring_flask_backend",
"remoteRoot": "/app"
}
],
"justMyCode": true
"name": "Run Metrics Server (TypeScript)",
"type": "node",
"request": "launch",
"runtimeExecutable": "bun",
"runtimeArgs": ["run"],
"program": "${workspaceFolder}/cli/lib/metrics-server.ts",
"env": {
"PORT": "8000",
"PROMETHEUS_URL": "http://127.0.0.1:59090"
},
"cwd": "${workspaceFolder}/cli",
"console": "integratedTerminal"
},
{
"name": "Run CLI Tests",
"type": "node",
"request": "launch",
"runtimeExecutable": "bun",
"runtimeArgs": ["test"],
"cwd": "${workspaceFolder}/cli",
"console": "integratedTerminal"
}
]
}


40 changes: 15 additions & 25 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ This workflow lets you:
- run the monitoring stack via Docker Compose
- iterate on **custom code** without rebuilding images or committing changes
- run the **reporter on your host** (recommended) and debug it
- optionally debug the Flask backend running in Docker
- optionally debug the metrics server running in Docker

### What runs where (quick mental model)

- **Docker**: pgwatch collectors + sinks + Grafana (+ optional Flask dev container)
- **Host**: `reporter/postgres_reports.py` (recommended for iteration & debugging)
- **Docker**: pgwatch collectors + sinks + Grafana (+ optional metrics-server dev container)
- **Host**: `cli/lib/reporter.ts` (recommended for iteration & debugging)

### One-time local setup (no commits)

Expand All @@ -53,7 +53,7 @@ cp docker-compose.override.example.yml docker-compose.override.yml
This enables:

- using local `./config/**` (Prometheus/Grafana/pgwatch configs) instead of published config images
- Flask bind-mount + optional debugpy
- Metrics server bind-mount for live reload
- exposing `sink-postgres` on localhost for host-run reporter
- (optional) an alternate mode to run the reporter *inside Docker* (commented in the example override). **Host-run reporter is the primary workflow.**

Expand Down Expand Up @@ -238,41 +238,31 @@ This repo includes `.vscode/launch.json` with a config:

Use **Run and Debug** → select **Run Reporter (local)**.

### Debug Flask backend in Docker (optional)
### Debug metrics-server in Docker (optional)

The override file bind-mounts `./monitoring_flask_backend` into the container for fast iteration.
The override file bind-mounts `./cli/lib` into the container for fast iteration.

#### Run without debugger
#### Run metrics-server locally

```bash
docker compose up -d --force-recreate monitoring_flask_backend
docker compose up -d --force-recreate metrics-server
```

#### Enable debugpy (attach debugger)

```bash
DEBUGPY_FLASK=1 docker compose up -d --force-recreate monitoring_flask_backend
```

Then attach from Cursor/VS Code:

- **Attach (Flask in Docker: debugpy 5678)**

The Flask service (gunicorn) is exposed on:
The metrics server is exposed on:

- `http://localhost:55000`

### (Optional) Debug reporter in Docker
### Debug reporter locally

This is usually slower than host-run debugging, but it exists for parity:
The reporter is a TypeScript/Bun application. Run it directly:

```bash
DEBUGPY_REPORTER=1 docker compose up -d --force-recreate postgres-reports
cd cli && bun run lib/reporter.ts --prometheus-url http://localhost:59090
```

Then attach:
Then use VS Code debugging:

- **Attach (Reporter in Docker: debugpy 5679)**
- **Run Reporter (TypeScript)** - from `.vscode/launch.example.json`

### Troubleshooting

Expand Down Expand Up @@ -341,7 +331,7 @@ postgresai mon reset
```bash
postgresai mon logs
postgresai mon logs grafana
postgresai mon logs monitoring_flask_backend
postgresai mon logs metrics-server
```

### Stop / start
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ This monitoring solution exposes several ports that **MUST** be properly firewal
- **Port 58089** (PGWatch Prometheus) - Database monitoring interface
- **Port 59090** (Victoria Metrics) - Metrics storage and queries
- **Port 59091** (PGWatch Prometheus endpoint) - Metrics collection
- **Port 55000** (Flask API) - Backend API service
- **Port 55000** (Metrics Server) - Backend API service
- **Port 55432** (Demo DB) - When using `--demo` option
- **Port 55433** (Metrics DB) - Postgres metrics storage

Expand Down
49 changes: 49 additions & 0 deletions cli/Dockerfile.metrics-server
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Dockerfile for the TypeScript Metrics Server
# Replaces the Python Flask backend with a Bun-based implementation

FROM oven/bun:1-slim AS builder

WORKDIR /app

# Copy package files
COPY package.json bun.lockb* ./

# Install dependencies
RUN bun install --frozen-lockfile

# Copy source files
COPY lib/ lib/
COPY bin/ bin/
COPY scripts/ scripts/
COPY test/ test/
COPY tsconfig.json ./

# Verify the build works
RUN bun run typecheck || true

# Production stage
FROM oven/bun:1-slim

# Install curl for healthcheck
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*

WORKDIR /app

# Copy node modules and source
COPY --from=builder /app/node_modules ./node_modules
COPY --from=builder /app/lib ./lib
COPY --from=builder /app/package.json ./

# Set environment variables
ENV PORT=8000
ENV PROMETHEUS_URL=http://localhost:8428

# Expose port
EXPOSE 8000

# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8000/health || exit 1

# Run the metrics server
CMD ["bun", "run", "lib/metrics-server.ts"]
42 changes: 42 additions & 0 deletions cli/Dockerfile.reporter
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Dockerfile for the TypeScript Reporter
# Replaces the Python reporter with a Bun-based implementation

FROM oven/bun:1-slim AS builder

WORKDIR /app

# Copy package files
COPY package.json bun.lockb* ./

# Install dependencies
RUN bun install --frozen-lockfile

# Copy source files
COPY lib/ lib/
COPY bin/ bin/
COPY scripts/ scripts/
COPY test/ test/
COPY tsconfig.json ./

# Verify the build works
RUN bun run typecheck || true

# Production stage
FROM oven/bun:1-slim

WORKDIR /app

# Copy node modules and source
COPY --from=builder /app/node_modules ./node_modules
COPY --from=builder /app/lib ./lib
COPY --from=builder /app/package.json ./

# Create reports directory
RUN mkdir -p /app/reports

# Set environment variables
ENV PROMETHEUS_URL=http://sink-prometheus:9090
ENV POSTGRES_SINK_URL=postgresql://pgwatch@sink-postgres:5432/measurements

# Run the reporter
CMD ["bun", "run", "lib/reporter.ts"]
2 changes: 1 addition & 1 deletion cli/bin/postgres-ai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1481,7 +1481,7 @@ mon
console.log(" ✅ PostgreSQL monitoring infrastructure");
console.log(" ✅ Grafana dashboards (with secure password)");
console.log(" ✅ Prometheus metrics storage");
console.log(" ✅ Flask API backend");
console.log(" ✅ Metrics server API");
console.log(" ✅ Automated report generation (every 24h)");
console.log(" ✅ Host stats monitoring (CPU, memory, disk, I/O)\n");

Expand Down
Loading