diff --git a/.env.sample b/.env.sample index fc3de809..2f8fdd74 100644 --- a/.env.sample +++ b/.env.sample @@ -1,6 +1,9 @@ # API server port PORT=4000 +# Metrics server port +METRICS_PORT=9090 + # Hawk API database URL MONGO_HAWK_DB_URL=mongodb://mongodb:27017/hawk diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3fcacf69..e58e0441 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,5 +10,10 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' + - name: Run tests run: yarn test:integration diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e30ca6e..211fe006 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,10 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version-file: '.nvmrc' - name: Install modules run: yarn - name: Run tests diff --git a/.gitignore b/.gitignore index b0e31773..1b1e52a6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ uploads globalConfig.json coverage tls +package-lock.json diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 3652b413..e48f560d 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,13 +1,13 @@ -FROM node:14.17.0-alpine as builder +FROM node:16-alpine as builder WORKDIR /usr/src/app -RUN apk add --no-cache git gcc g++ python make musl-dev +RUN apk add --no-cache git gcc g++ python3 make musl-dev COPY package.json yarn.lock ./ RUN yarn install -FROM node:14.17.0-alpine +FROM node:16-alpine WORKDIR /usr/src/app diff --git a/docs/METRICS.md b/docs/METRICS.md new file mode 100644 index 00000000..76c76b47 --- /dev/null +++ b/docs/METRICS.md @@ -0,0 +1,120 @@ +# Prometheus Metrics + +This application exposes Prometheus-compatible metrics on a separate port from the main API server. + +## Configuration + +The metrics server runs on a separate port configured via the `METRICS_PORT` environment variable: + +```bash +# Default: 9090 +METRICS_PORT=9090 +``` + +Add this to your `.env` file. See `.env.sample` for reference. + +## Metrics Endpoint + +The metrics are served at: + +``` +http://localhost:9090/metrics +``` + +(Replace `9090` with your configured `METRICS_PORT` if different) + +## Available Metrics + +### Default Node.js Metrics + +The following default Node.js metrics are automatically collected: + +- **nodejs_version_info** - Node.js version information +- **process_cpu_user_seconds_total** - Total user CPU time spent in seconds +- **process_cpu_system_seconds_total** - Total system CPU time spent in seconds +- **nodejs_heap_size_total_bytes** - Total heap size in bytes +- **nodejs_heap_size_used_bytes** - Used heap size in bytes +- **nodejs_external_memory_bytes** - External memory in bytes +- **nodejs_heap_space_size_total_bytes** - Total heap space size in bytes +- **nodejs_heap_space_size_used_bytes** - Used heap space size in bytes +- **nodejs_eventloop_lag_seconds** - Event loop lag in seconds +- **nodejs_eventloop_lag_min_seconds** - Minimum event loop lag +- **nodejs_eventloop_lag_max_seconds** - Maximum event loop lag +- **nodejs_eventloop_lag_mean_seconds** - Mean event loop lag +- **nodejs_eventloop_lag_stddev_seconds** - Standard deviation of event loop lag +- **nodejs_eventloop_lag_p50_seconds** - 50th percentile event loop lag +- **nodejs_eventloop_lag_p90_seconds** - 90th percentile event loop lag +- **nodejs_eventloop_lag_p99_seconds** - 99th percentile event loop lag + +### Custom HTTP Metrics + +#### http_request_duration_seconds (Histogram) + +Duration of HTTP requests in seconds, labeled by: +- `method` - HTTP method (GET, POST, etc.) +- `route` - Request route/path +- `status_code` - HTTP status code + +Buckets: 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10 seconds + +#### http_requests_total (Counter) + +Total number of HTTP requests, labeled by: +- `method` - HTTP method (GET, POST, etc.) +- `route` - Request route/path +- `status_code` - HTTP status code + +## Testing + +### Manual Testing + +You can test the metrics endpoint using curl: + +```bash +curl http://localhost:9090/metrics +``` + +Or run the provided test script: + +```bash +./test-metrics.sh +``` + +### Integration Tests + +Integration tests for metrics are located in `test/integration/cases/metrics.test.ts`. + +Run them with: + +```bash +npm run test:integration +``` + +## Implementation Details + +The metrics implementation uses the `prom-client` library and consists of: + +1. **Metrics Module** (`src/metrics/index.ts`): + - Initializes a Prometheus registry + - Configures default Node.js metrics collection + - Defines custom HTTP metrics (duration histogram and request counter) + - Provides middleware for tracking HTTP requests + - Creates a separate Express app for serving metrics + +2. **Integration** (`src/index.ts`): + - Adds metrics middleware to the main Express app + - Starts metrics server on a separate port + - Keeps metrics server isolated from main API traffic + +## Prometheus Configuration + +To scrape these metrics with Prometheus, add the following to your `prometheus.yml`: + +```yaml +scrape_configs: + - job_name: 'hawk-api' + static_configs: + - targets: ['localhost:9090'] +``` + +Adjust the target host and port according to your deployment. diff --git a/package.json b/package.json index a4748169..f4b8a889 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.1.41", + "version": "1.1.42", "main": "index.ts", "license": "BUSL-1.1", "scripts": { @@ -80,6 +80,7 @@ "mime-types": "^2.1.25", "mongodb": "^3.7.3", "morgan": "^1.10.1", + "prom-client": "^15.1.3", "safe-regex": "^2.1.0", "ts-node-dev": "^2.0.0", "uuid": "^8.3.2" diff --git a/src/index.ts b/src/index.ts index 00b157c7..17ae5d26 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,6 +27,7 @@ import BusinessOperationsFactory from './models/businessOperationsFactory'; import schema from './schema'; import { graphqlUploadExpress } from 'graphql-upload'; import morgan from 'morgan'; +import { metricsMiddleware, createMetricsServer } from './metrics'; /** * Option to enable playground @@ -48,6 +49,11 @@ class HawkAPI { */ private serverPort = +(process.env.PORT || 4000); + /** + * Port to serve metrics endpoint + */ + private metricsPort = +(process.env.METRICS_PORT || 9090); + /** * Express application */ @@ -86,6 +92,11 @@ class HawkAPI { */ this.app.use(morgan(process.env.NODE_ENV === 'production' ? 'combined' : 'dev')); + /** + * Add metrics middleware to track HTTP requests + */ + this.app.use(metricsMiddleware); + this.app.use(express.json()); this.app.use(bodyParser.urlencoded({ extended: false })); this.app.use('/static', express.static(`./static`)); @@ -241,6 +252,15 @@ class HawkAPI { this.app.use(graphqlUploadExpress()); this.server.applyMiddleware({ app: this.app }); + // Start metrics server on separate port + const metricsApp = createMetricsServer(); + + metricsApp.listen(this.metricsPort, () => { + console.log( + `📊 Metrics server ready at http://localhost:${this.metricsPort}/metrics` + ); + }); + return new Promise((resolve) => { this.httpServer.listen({ port: this.serverPort }, () => { console.log( diff --git a/src/metrics/index.ts b/src/metrics/index.ts new file mode 100644 index 00000000..14818dd0 --- /dev/null +++ b/src/metrics/index.ts @@ -0,0 +1,73 @@ +import client from 'prom-client'; +import express from 'express'; + +/** + * Create a Registry to register the metrics + */ +const register = new client.Registry(); + +/** + * Add default Node.js metrics (CPU, memory, event loop, etc.) + */ +client.collectDefaultMetrics({ register }); + +/** + * HTTP request duration histogram + * Tracks request duration by route, method, and status code + */ +const httpRequestDuration = new client.Histogram({ + name: 'http_request_duration_seconds', + help: 'Duration of HTTP requests in seconds', + labelNames: ['method', 'route', 'status_code'], + buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10], + registers: [ register ], +}); + +/** + * HTTP request counter + * Tracks count of HTTP requests by route, method, and status code + */ +const httpRequestCounter = new client.Counter({ + name: 'http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'route', 'status_code'], + registers: [ register ], +}); + +/** + * Express middleware to track HTTP metrics + */ +export function metricsMiddleware(req: express.Request, res: express.Response, next: express.NextFunction): void { + const start = Date.now(); + + // Hook into response finish event to capture metrics + res.on('finish', () => { + const duration = (Date.now() - start) / 1000; // Convert to seconds + const route = req.route ? req.route.path : req.path; + const method = req.method; + const statusCode = res.statusCode.toString(); + + // Record metrics + httpRequestDuration.labels(method, route, statusCode).observe(duration); + httpRequestCounter.labels(method, route, statusCode).inc(); + }); + + next(); +} + +/** + * Create metrics server + * @returns Express application serving metrics endpoint + */ +export function createMetricsServer(): express.Application { + const metricsApp = express(); + + metricsApp.get('/metrics', async (req, res) => { + res.setHeader('Content-Type', register.contentType); + const metrics = await register.metrics(); + + res.send(metrics); + }); + + return metricsApp; +} diff --git a/src/types/env.d.ts b/src/types/env.d.ts index 7b98a1bc..82eb4ce9 100644 --- a/src/types/env.d.ts +++ b/src/types/env.d.ts @@ -11,6 +11,11 @@ declare namespace NodeJS { */ PORT: string; + /** + * Metrics server port + */ + METRICS_PORT: string; + /** * MongoDB url */ diff --git a/test-metrics.sh b/test-metrics.sh new file mode 100755 index 00000000..74e2fd9f --- /dev/null +++ b/test-metrics.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Script to manually test the Prometheus metrics endpoint +# This can be run locally with a running instance of the API + +METRICS_PORT=${METRICS_PORT:-9090} +METRICS_URL="http://localhost:${METRICS_PORT}/metrics" + +echo "Testing Prometheus Metrics Endpoint..." +echo "URL: ${METRICS_URL}" +echo "" + +# Test if the endpoint is accessible +if curl -s -o /dev/null -w "%{http_code}" "${METRICS_URL}" | grep -q "200"; then + echo "✓ Metrics endpoint is accessible (HTTP 200)" +else + echo "✗ Metrics endpoint is not accessible" + exit 1 +fi + +echo "" +echo "Sample metrics output:" +echo "======================" +curl -s "${METRICS_URL}" | head -50 +echo "" +echo "..." +echo "" + +# Check for specific metrics +echo "Checking for required metrics..." + +if curl -s "${METRICS_URL}" | grep -q "nodejs_version_info"; then + echo "✓ Default Node.js metrics present" +else + echo "✗ Default Node.js metrics missing" + exit 1 +fi + +if curl -s "${METRICS_URL}" | grep -q "http_request_duration_seconds"; then + echo "✓ HTTP request duration metrics present" +else + echo "✗ HTTP request duration metrics missing" + exit 1 +fi + +if curl -s "${METRICS_URL}" | grep -q "http_requests_total"; then + echo "✓ HTTP request counter metrics present" +else + echo "✗ HTTP request counter metrics missing" + exit 1 +fi + +echo "" +echo "All checks passed! ✓" diff --git a/test/integration/cases/metrics.test.ts b/test/integration/cases/metrics.test.ts new file mode 100644 index 00000000..c78e9997 --- /dev/null +++ b/test/integration/cases/metrics.test.ts @@ -0,0 +1,41 @@ +import axios from 'axios'; +import dotenv from 'dotenv'; +import path from 'path'; + +/** + * Env variables for API + */ +const apiEnv = dotenv.config({ path: path.join(__dirname, '../api.env') }).parsed || {}; + +/** + * Axios instance to send requests to metrics endpoint + */ +const metricsInstance = axios.create({ + baseURL: `http://api:${apiEnv.METRICS_PORT || 9090}`, +}); + +describe('Prometheus Metrics', () => { + test('Metrics endpoint is accessible', async () => { + const response = await metricsInstance.get('/metrics'); + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toMatch(/text\/plain/); + }); + + test('Metrics endpoint returns prometheus format', async () => { + const response = await metricsInstance.get('/metrics'); + + // Check for some default Node.js metrics + expect(response.data).toContain('nodejs_version_info'); + expect(response.data).toContain('process_cpu_user_seconds_total'); + expect(response.data).toContain('nodejs_heap_size_total_bytes'); + }); + + test('Metrics endpoint includes custom HTTP metrics', async () => { + const response = await metricsInstance.get('/metrics'); + + // Check for our custom metrics + expect(response.data).toContain('http_request_duration_seconds'); + expect(response.data).toContain('http_requests_total'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 53e65c77..deea0104 100644 --- a/yarn.lock +++ b/yarn.lock @@ -725,6 +725,11 @@ resolved "https://registry.yarnpkg.com/@n1ru4l/json-patch-plus/-/json-patch-plus-0.2.0.tgz#b8fa09fd980c3460dfdc109a7c4cc5590157aa6b" integrity sha512-pLkJy83/rVfDTyQgDSC8GeXAHEdXNHGNJrB1b7wAyGQu0iv7tpMXntKVSqj0+XKNVQbco40SZffNfVALzIt0SQ== +"@opentelemetry/api@^1.4.0": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@phc/format@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@phc/format/-/format-1.0.0.tgz#b5627003b3216dc4362125b13f48a4daa76680e4" @@ -1781,6 +1786,11 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + bitsyntax@~0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/bitsyntax/-/bitsyntax-0.1.0.tgz#b0c59acef03505de5a2ed62a2f763c56ae1d6205" @@ -5523,6 +5533,14 @@ progress@^2.0.0: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +prom-client@^15.1.3: + version "15.1.3" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2" + integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + promise-breaker@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/promise-breaker/-/promise-breaker-5.0.0.tgz#58e8541f1619554057da95a211794d7834d30c1d" @@ -6444,6 +6462,13 @@ tar@^6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + terminal-link@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/terminal-link/-/terminal-link-2.1.1.tgz#14a64a27ab3c0df933ea546fba55f2d078edc994"