Skip to content

Commit f81ce0a

Browse files
deepakdgupta1deepakdgupta1
committed
feat: Implement Agent Gateway with local setup, CI/CD, operational guides, and add API rate limiting.
1 parent f7ed623 commit f81ce0a

26 files changed

Lines changed: 1353 additions & 34 deletions

‎.github/workflows/agent-gateway-ci.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Agent Gateway CI
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "apps/agent-gateway/**"
7+
- ".github/workflows/agent-gateway-ci.yml"
8+
push:
9+
branches: [main]
10+
paths:
11+
- "apps/agent-gateway/**"
12+
- ".github/workflows/agent-gateway-ci.yml"
13+
workflow_dispatch:
14+
15+
permissions:
16+
contents: read
17+
18+
jobs:
19+
lint-and-test:
20+
name: Lint and test gateway
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: Checkout
24+
uses: actions/checkout@v4
25+
26+
- name: Install uv
27+
uses: astral-sh/setup-uv@v3
28+
with:
29+
enable-cache: true
30+
cache-dependency-glob: "apps/agent-gateway/uv.lock"
31+
32+
- name: Install gateway dependencies
33+
run: uv sync --project apps/agent-gateway --extra dev
34+
35+
- name: Ruff
36+
run: uv run --project apps/agent-gateway ruff check src tests scripts
37+
working-directory: apps/agent-gateway
38+
39+
- name: Pytest
40+
run: uv run --project apps/agent-gateway pytest -q tests
41+
working-directory: apps/agent-gateway
42+
43+
docker-build:
44+
name: Build gateway image
45+
runs-on: ubuntu-latest
46+
needs: lint-and-test
47+
steps:
48+
- name: Checkout
49+
uses: actions/checkout@v4
50+
51+
- name: Set up Docker Buildx
52+
uses: docker/setup-buildx-action@v3
53+
54+
- name: Build image
55+
uses: docker/build-push-action@v5
56+
with:
57+
context: apps/agent-gateway
58+
file: apps/agent-gateway/Dockerfile
59+
tags: knowcode-agent-gateway:ci
60+
push: false

‎.github/workflows/agent-gateway-smoke.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Agent Gateway Smoke
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
gateway_url:
7+
description: "Gateway base URL (for example https://gateway.example.com)"
8+
required: true
9+
type: string
10+
min_tool_calls:
11+
description: "Minimum successful tool executions"
12+
required: false
13+
default: "1"
14+
type: string
15+
tool_names:
16+
description: "Optional comma-separated tool names"
17+
required: false
18+
default: ""
19+
type: string
20+
21+
permissions:
22+
contents: read
23+
24+
jobs:
25+
smoke:
26+
runs-on: ubuntu-latest
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
31+
- name: Run smoke test
32+
run: |
33+
set -euo pipefail
34+
ARGS=(
35+
--gateway-url "${{ inputs.gateway_url }}"
36+
--min-tool-calls "${{ inputs.min_tool_calls }}"
37+
)
38+
if [ -n "${{ inputs.tool_names }}" ]; then
39+
ARGS+=(--tool-names "${{ inputs.tool_names }}")
40+
fi
41+
python3 apps/agent-gateway/scripts/smoke_e2e.py "${ARGS[@]}"

‎MULTI_AGENT_SETUP_EFFICIENCY.md‎

Lines changed: 168 additions & 12 deletions
Large diffs are not rendered by default.

‎apps/agent-gateway/.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
KNOWCODE_API_BASE_URL=http://127.0.0.1:8000
22
LITELLM_BASE_URL=http://127.0.0.1:4000
3+
LITELLM_MASTER_KEY=sk-local-proxy
34
LITELLM_API_KEY=sk-local-proxy
45
AGENT_DEFAULT_MODEL=gemini/gemini-3-flash-preview
56
AGENT_MAX_TOOL_ROUNDS=4
67
AGENT_TOOL_TIMEOUT_SECONDS=30
78
AGENT_OPENAPI_CACHE_TTL_SECONDS=300
89
AGENT_ALLOWED_TOOL_NAMES=query_context,search,get_context,trace_calls
910
AGENT_DEFAULT_TAGS=knowcode,context
11+
AGENT_STRICT_ENV_VALIDATION=false
12+
LOG_LEVEL=INFO
1013
HOST=127.0.0.1
1114
PORT=8081

‎apps/agent-gateway/.env.production.example‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Production template for agent-gateway.
2+
# Keep this file as a template only; store real values in your secret manager.
3+
4+
KNOWCODE_API_BASE_URL=https://knowcode-api.example.com
5+
LITELLM_BASE_URL=https://litellm.example.com
6+
LITELLM_MASTER_KEY=__SET_IN_SECRET_MANAGER__
7+
LITELLM_API_KEY=__SET_IN_SECRET_MANAGER__
8+
9+
AGENT_DEFAULT_MODEL=gemini/gemini-3-flash-preview
10+
AGENT_MAX_TOOL_ROUNDS=4
11+
AGENT_TOOL_TIMEOUT_SECONDS=30
12+
AGENT_OPENAPI_CACHE_TTL_SECONDS=300
13+
AGENT_ALLOWED_TOOL_NAMES=query_context,search,get_context,trace_calls
14+
AGENT_DEFAULT_TAGS=knowcode,context,prod
15+
AGENT_STRICT_ENV_VALIDATION=true
16+
LOG_LEVEL=INFO
17+
18+
HOST=0.0.0.0
19+
PORT=8081

‎apps/agent-gateway/OPERATIONS.md‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Agent Gateway Operations Runbook
2+
3+
## 1) Preflight
4+
5+
1. Confirm KnowCode API and LiteLLM URLs are reachable from gateway runtime.
6+
2. Provision secrets in your secret manager:
7+
- `LITELLM_API_KEY`
8+
- upstream model provider keys used by LiteLLM (for example `GOOGLE_API_KEY_1`)
9+
3. Use `AGENT_STRICT_ENV_VALIDATION=true` in non-local environments.
10+
4. Restrict tool scope with `AGENT_ALLOWED_TOOL_NAMES`.
11+
12+
### Local self-host mode
13+
14+
1. Export env vars on your machine:
15+
- `GOOGLE_API_KEY_1`
16+
- `LITELLM_MASTER_KEY`
17+
- `LITELLM_API_KEY` (typically same as `LITELLM_MASTER_KEY`)
18+
2. Run `apps/agent-gateway/scripts/local_up.sh`.
19+
3. Run smoke test:
20+
- `uv run --project apps/agent-gateway python apps/agent-gateway/scripts/smoke_e2e.py --gateway-url http://127.0.0.1:8081`
21+
22+
## 2) CI Gate
23+
24+
CI workflow: `.github/workflows/agent-gateway-ci.yml`
25+
26+
1. Lint: `ruff check apps/agent-gateway/src apps/agent-gateway/tests apps/agent-gateway/scripts`
27+
2. Tests: `pytest apps/agent-gateway/tests`
28+
3. Docker build validation (`apps/agent-gateway/Dockerfile`)
29+
30+
## 3) Deploy
31+
32+
1. Deploy gateway as its own service/container.
33+
2. Configure readiness probe:
34+
- `GET /ready` (checks KnowCode API, LiteLLM, and OpenAPI tool translation)
35+
3. Configure liveness probe:
36+
- `GET /health`
37+
4. Roll out with rolling deployment by default.
38+
39+
## 4) Post-Deploy Smoke
40+
41+
Use `.github/workflows/agent-gateway-smoke.yml` or run manually:
42+
43+
```bash
44+
uv run --project apps/agent-gateway python apps/agent-gateway/scripts/smoke_e2e.py \
45+
--gateway-url https://your-gateway.example.com \
46+
--min-tool-calls 1
47+
```
48+
49+
## 5) Monitoring and Alerts
50+
51+
The gateway emits JSON logs for:
52+
53+
1. `http_request` with request path, status code, and latency.
54+
2. `chat_completed` with model, tool counts, response cost, and token usage.
55+
3. `chat_failed` with failure type and error detail.
56+
57+
Set alerts on:
58+
59+
1. `GET /ready` failures.
60+
2. Elevated `chat_failed` rate.
61+
3. Tool timeout spikes (`failed_tool_calls` or high `total_tool_latency_ms`).
62+
4. Cost anomalies (`response_cost` trends).
63+
64+
## 6) Rollback
65+
66+
1. Keep prior image digest available.
67+
2. If readiness or error-rate SLO is breached, roll back to previous stable image.
68+
3. Re-run smoke against rolled-back version.

‎apps/agent-gateway/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,54 @@ uv run --project apps/agent-gateway agent-gateway
3838

3939
Gateway endpoints:
4040
- `GET /health`
41+
- `GET /ready`
4142
- `GET /api/v1/tools`
4243
- `POST /api/v1/chat`
4344

45+
## Local Self-Hosted Mode (KnowCode + Gateway on your machine)
46+
47+
You can run everything locally with one command:
48+
49+
```bash
50+
# Required: export keys in your shell profile or current terminal
51+
export GOOGLE_API_KEY_1="..."
52+
export LITELLM_MASTER_KEY="sk-your-local-master-key"
53+
export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
54+
55+
# Optional override (default works for local Docker + local KnowCode)
56+
export KNOWCODE_API_BASE_URL="http://host.docker.internal:8000"
57+
58+
apps/agent-gateway/scripts/local_up.sh
59+
```
60+
61+
Stop all services:
62+
63+
```bash
64+
apps/agent-gateway/scripts/local_down.sh
65+
```
66+
67+
Notes:
68+
- `local_up.sh` starts KnowCode API on `0.0.0.0:8000` if it is not already running.
69+
- It then starts LiteLLM + gateway via Docker Compose and waits for `/ready`.
70+
- Ollama fallback remains configured. If Ollama is installed and running on your host (`:11434`), LiteLLM can route to it.
71+
72+
## End-to-End Smoke Test
73+
74+
With KnowCode API, LiteLLM, and this gateway running, execute:
75+
76+
```bash
77+
uv run --project apps/agent-gateway python scripts/smoke_e2e.py
78+
```
79+
80+
Optional strict tool set:
81+
82+
```bash
83+
uv run --project apps/agent-gateway python scripts/smoke_e2e.py \
84+
--tool-names query_context,get_context \
85+
--min-tool-calls 1 \
86+
--print-json
87+
```
88+
4489
## Docker Run
4590

4691
```bash
@@ -51,6 +96,21 @@ docker compose up --build
5196
This starts LiteLLM + the gateway. KnowCode is expected at
5297
`http://host.docker.internal:8000` by default.
5398

99+
## Production Notes
100+
101+
- Use `AGENT_STRICT_ENV_VALIDATION=true` for non-local environments.
102+
- Keep secrets in a secret manager, not checked-in files.
103+
- Use `.env.production.example` as a template only.
104+
- Read `OPERATIONS.md` for rollout, readiness probes, monitoring, and rollback.
105+
106+
## CI Workflows
107+
108+
- `.github/workflows/agent-gateway-ci.yml`
109+
- Runs Ruff + pytest for `apps/agent-gateway`
110+
- Validates Docker build
111+
- `.github/workflows/agent-gateway-smoke.yml`
112+
- Manual post-deploy smoke test against a live gateway URL
113+
54114
## Extraction Checklist (To New Repo)
55115

56116
1. Copy `apps/agent-gateway` into a new repository root.

‎apps/agent-gateway/docker-compose.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ services:
88
GOOGLE_API_KEY_1: ${GOOGLE_API_KEY_1}
99
volumes:
1010
- ./litellm.config.yaml:/app/config.yaml:ro
11+
extra_hosts:
12+
- "host.docker.internal:host-gateway"
1113
ports:
1214
- "4000:4000"
1315

@@ -18,6 +20,9 @@ services:
1820
LITELLM_BASE_URL: http://litellm:4000
1921
LITELLM_API_KEY: ${LITELLM_MASTER_KEY:-sk-local-proxy}
2022
AGENT_DEFAULT_MODEL: ${AGENT_DEFAULT_MODEL:-gemini/gemini-3-flash-preview}
23+
AGENT_STRICT_ENV_VALIDATION: ${AGENT_STRICT_ENV_VALIDATION:-false}
24+
LOG_LEVEL: ${LOG_LEVEL:-INFO}
25+
HOST: 0.0.0.0
2126
PORT: 8081
2227
depends_on:
2328
- litellm

‎apps/agent-gateway/scripts/local_down.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
6+
GATEWAY_DIR="${PROJECT_ROOT}/apps/agent-gateway"
7+
8+
echo "Stopping Docker Compose services"
9+
cd "${GATEWAY_DIR}"
10+
docker compose down || true
11+
12+
if [[ -f /tmp/knowcode-api.pid ]]; then
13+
PID="$(cat /tmp/knowcode-api.pid || true)"
14+
if [[ -n "${PID}" ]] && kill -0 "${PID}" >/dev/null 2>&1; then
15+
echo "Stopping KnowCode API process ${PID}"
16+
kill "${PID}" || true
17+
fi
18+
rm -f /tmp/knowcode-api.pid
19+
fi
20+
21+
echo "Local stack stopped."

‎apps/agent-gateway/scripts/local_up.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
6+
GATEWAY_DIR="${PROJECT_ROOT}/apps/agent-gateway"
7+
KNOWCODE_VENV_PYTHON="${PROJECT_ROOT}/.venv/bin/python"
8+
9+
export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-local-proxy}"
10+
export LITELLM_API_KEY="${LITELLM_API_KEY:-${LITELLM_MASTER_KEY}}"
11+
export KNOWCODE_API_BASE_URL="${KNOWCODE_API_BASE_URL:-http://host.docker.internal:8000}"
12+
13+
if ! command -v uv >/dev/null 2>&1; then
14+
echo "uv is required but not installed." >&2
15+
exit 1
16+
fi
17+
18+
if ! command -v docker >/dev/null 2>&1; then
19+
echo "docker is required but not installed." >&2
20+
exit 1
21+
fi
22+
23+
if ! ss -ltn | rg -q ':8000'; then
24+
echo "Starting KnowCode API on 0.0.0.0:8000"
25+
# Use setsid so the API process survives after this script exits.
26+
if [[ -x "${KNOWCODE_VENV_PYTHON}" ]]; then
27+
setsid env PYTHONPATH="${PROJECT_ROOT}/src:${PYTHONPATH:-}" \
28+
"${KNOWCODE_VENV_PYTHON}" -m uvicorn knowcode.api.main:create_app \
29+
--factory --host 0.0.0.0 --port 8000 > /tmp/knowcode-api.log 2>&1 < /dev/null &
30+
else
31+
setsid uv run uvicorn knowcode.api.main:create_app --factory --host 0.0.0.0 --port 8000 \
32+
> /tmp/knowcode-api.log 2>&1 < /dev/null &
33+
fi
34+
echo "$!" > /tmp/knowcode-api.pid
35+
else
36+
echo "KnowCode API already listening on :8000"
37+
fi
38+
39+
echo "Waiting for KnowCode OpenAPI..."
40+
for _ in $(seq 1 30); do
41+
if curl -fsS "http://127.0.0.1:8000/openapi.json" >/dev/null; then
42+
break
43+
fi
44+
sleep 1
45+
done
46+
47+
if ! curl -fsS "http://127.0.0.1:8000/openapi.json" >/dev/null; then
48+
echo "KnowCode API did not become ready. Check /tmp/knowcode-api.log" >&2
49+
exit 1
50+
fi
51+
52+
echo "Starting LiteLLM + Agent Gateway with Docker Compose"
53+
cd "${GATEWAY_DIR}"
54+
if docker buildx version >/dev/null 2>&1; then
55+
docker compose up -d --build
56+
else
57+
DOCKER_BUILDKIT=0 docker compose up -d --build
58+
fi
59+
60+
echo "Waiting for gateway readiness..."
61+
for _ in $(seq 1 45); do
62+
if curl -fsS "http://127.0.0.1:8081/ready" >/dev/null; then
63+
break
64+
fi
65+
sleep 1
66+
done
67+
68+
curl -fsS "http://127.0.0.1:8081/ready" >/dev/null
69+
echo "Local stack is ready:"
70+
echo " KnowCode API: http://127.0.0.1:8000"
71+
echo " AI Gateway: http://127.0.0.1:8081"

0 commit comments

Comments
 (0)