Skip to content

Commit 2b09602

Browse files
James Youngbloodmarkharding
James Youngblood
authored and
markharding
committed
[#95] Serve label events and allow filtering of label targets at request
changelog: enhancement
1 parent 817e7dc commit 2b09602

28 files changed

Lines changed: 2145 additions & 177 deletions

‎.gitlab-ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ include:
1111
- local: '.gitlab/ci/deploy_services.yml'
1212
- local: '.gitlab/ci/deploy_pages.yml'
1313
- local: '.gitlab/ci/scan_dependencies.yml'
14+
- local: '.gitlab/ci/test_server.yml'
1415
- local: '.gitlab/ci/test_moderation.yml'
1516
- local: '.gitlab/ci/release.yml'
1617
stages:

‎.gitlab/ci/scripts/integration-moderation.sh‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,6 @@ if [[ "${CI:-}" == "true" ]]; then
6161
# Reach the services by their in-network names instead of localhost. Kafka
6262
# is reached on its INTERNAL listener, which advertises kafka:19092 on this
6363
# network (the EXTERNAL listener advertises localhost:9092, for local use).
64-
SERVER_HOST=server
65-
export POLYCENTRIC_TEST_SERVER="http://server:3000"
6664
export POLYCENTRIC_TEST_DATABASE_URL="postgres://postgres:testing@postgres:5432"
6765
export POLYCENTRIC_TEST_OS_ENDPOINT="http://rustfs:9000"
6866
export POLYCENTRIC_TEST_KAFKA_BROKERS="kafka:19092"
@@ -74,13 +72,30 @@ echo "==> Building and starting the server…"
7472
# running, so this is safe.
7573
docker compose up -d --no-deps --build --wait server
7674

75+
# Resolve the server container's IP on the compose network and use it
76+
# directly, bypassing Docker embedded DNS (which can be flaky in a
77+
# Docker-in-Docker environment when the job container is connected to the
78+
# compose network via `docker network connect`).
79+
if [[ "${CI:-}" == "true" ]]; then
80+
SERVER_HOST=server
81+
SERVER_IP=$(docker inspect -f '{{(index .NetworkSettings.Networks "'${NETWORK}'").IPAddress}}' polycentric-server-1 2>/dev/null)
82+
if [ -n "$SERVER_IP" ]; then
83+
SERVER_HOST=$SERVER_IP
84+
fi
85+
fi
86+
export POLYCENTRIC_TEST_SERVER="http://${SERVER_HOST}:${SERVER_PORT}"
87+
7788
echo "==> Waiting for the server gRPC port ($SERVER_HOST:$SERVER_PORT)…"
78-
for _ in $(seq 1 60); do
89+
for i in $(seq 1 60); do
7990
if (exec 3<>"/dev/tcp/$SERVER_HOST/$SERVER_PORT") 2>/dev/null; then
8091
exec 3>&- 3<&-
81-
echo " server is accepting connections"
92+
echo " server is accepting connections (after ${i}s)"
8293
break
8394
fi
95+
if [ "$i" -eq 60 ]; then
96+
echo "ERROR: server did not start within 60 seconds"
97+
exit 1
98+
fi
8499
sleep 1
85100
done
86101

‎.gitlab/ci/scripts/integration-server.sh‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Run the server integration test suite.
4+
#
5+
# Starts PostgreSQL + RustFS + Kafka (if not already running), applies
6+
# migrations, starts the server with a deterministic test moderator identity,
7+
# runs `cargo test -p integration-tests`, then cleans up.
8+
#
9+
# Usage:
10+
# .gitlab/ci/scripts/integration-server.sh # full run
11+
# .gitlab/ci/scripts/integration-server.sh --no-deps # skip docker services (already up)
12+
# .gitlab/ci/scripts/integration-server.sh --no-cleanup # keep server + docker running
13+
# .gitlab/ci/scripts/integration-server.sh --ci # CI mode
14+
#
15+
# Run the server integration test suite.
16+
#
17+
# Starts PostgreSQL + RustFS + Kafka (if not already running), applies
18+
# migrations, starts the server with a deterministic test moderator identity,
19+
# runs `cargo test -p integration-tests`, then cleans up.
20+
#
21+
22+
set -euo pipefail
23+
24+
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
25+
cd "$REPO_ROOT"
26+
export COMPOSE_PROJECT_NAME=polycentric
27+
28+
# ---------------------------------------------------------------------------
29+
# Test moderator identity (deterministic — derived from the same seed as
30+
# `test_moderator_key()` in the integration-test crate). Generated via:
31+
# seed = sha256(b"polycentric-test-moderator-seed-2026")
32+
# key = Ed25519SigningKey::from_bytes(&seed)
33+
# Identity{rotation_keys=[PublicKey{type=Ed25519, key=pub}]} →
34+
# hex(sha256(prost::encode(identity)))
35+
# ---------------------------------------------------------------------------
36+
MODERATOR_IDENTITY="020225a394cac01413ff43527f1644b1772d78d2cea873de1e8ae2f9c3c9f47b"
37+
38+
NO_DEPS=false
39+
NO_CLEANUP=false
40+
CI_MODE=false
41+
42+
for arg in "$@"; do
43+
case "$arg" in
44+
--no-deps) NO_DEPS=true ;;
45+
--no-cleanup) NO_CLEANUP=true ;;
46+
--ci) CI_MODE=true ;;
47+
esac
48+
done
49+
50+
# ---------------------------------------------------------------------------
51+
# Cleanup trap
52+
# ---------------------------------------------------------------------------
53+
cleanup() {
54+
if [ "$NO_CLEANUP" = true ]; then
55+
return
56+
fi
57+
echo ""
58+
echo "==> Cleaning up…"
59+
if [ -n "${SERVER_PID:-}" ]; then
60+
kill "$SERVER_PID" 2>/dev/null || true
61+
wait "$SERVER_PID" 2>/dev/null || true
62+
echo " server stopped (PID $SERVER_PID)"
63+
fi
64+
if [ "$NO_DEPS" = false ]; then
65+
echo " stopping docker compose services…"
66+
docker compose down -v >/dev/null 2>&1 || true
67+
fi
68+
}
69+
trap cleanup EXIT INT TERM
70+
71+
# Pre-flight: purge any stale stack from a hard-killed prior job so we start
72+
# with a clean slate (volumes, network, broker state).
73+
docker compose down -v >/dev/null 2>&1 || true
74+
75+
# ---------------------------------------------------------------------------
76+
# 1. Backing services (PostgreSQL, RustFS, Kafka)
77+
# ---------------------------------------------------------------------------
78+
if [ "$NO_DEPS" = false ]; then
79+
echo "==> Starting PostgreSQL, RustFS, and Kafka…"
80+
if [ "$CI_MODE" = true ]; then
81+
# CI: docker compose builds the image explicitly; --wait blocks until all
82+
# services are healthy.
83+
docker compose up -d --build --wait postgres rustfs kafka
84+
else
85+
docker compose up -d postgres rustfs rustfs-init kafka
86+
87+
echo " waiting for postgres (port 5432)…"
88+
for i in $(seq 1 30); do
89+
if (exec 3<>"/dev/tcp/localhost/5432") 2>/dev/null; then
90+
exec 3>&- 3<&-
91+
echo " postgres ready (after ${i}s)"
92+
break
93+
fi
94+
if [ "$i" -eq 30 ]; then
95+
echo "ERROR: postgres not available within 30 seconds"
96+
exit 1
97+
fi
98+
sleep 1
99+
done
100+
101+
echo " waiting for RustFS (port 9000)…"
102+
for i in $(seq 1 15); do
103+
if curl -sf http://localhost:9000/health > /dev/null 2>&1; then
104+
echo " RustFS ready (after ${i}s)"
105+
break
106+
fi
107+
if [ "$i" -eq 15 ]; then
108+
echo "ERROR: RustFS not available within 15 seconds"
109+
exit 1
110+
fi
111+
sleep 1
112+
done
113+
114+
echo " waiting for Kafka (port 9092)…"
115+
for i in $(seq 1 30); do
116+
if (exec 3<>"/dev/tcp/localhost/9092") 2>/dev/null; then
117+
exec 3>&- 3<&-
118+
echo " Kafka ready (after ${i}s)"
119+
break
120+
fi
121+
if [ "$i" -eq 30 ]; then
122+
echo "ERROR: Kafka not available within 30 seconds"
123+
exit 1
124+
fi
125+
sleep 1
126+
done
127+
fi
128+
echo " backing services ready"
129+
fi
130+
131+
# ---------------------------------------------------------------------------
132+
# 2. Start server (CI) or start server and run migrations (local)
133+
# ---------------------------------------------------------------------------
134+
if [ "$CI_MODE" = true ]; then
135+
NETWORK="${COMPOSE_PROJECT_NAME}_default"
136+
137+
self_container() {
138+
local id
139+
id=$(grep -oE 'containers/[0-9a-f]{64}' /proc/self/mountinfo | head -1 | cut -d/ -f2)
140+
echo "${id:-$(cat /etc/hostname)}"
141+
}
142+
143+
echo "==> Joining job container to the stack network ($NETWORK)…"
144+
docker network connect "$NETWORK" "$(self_container)"
145+
146+
echo "==> Starting server…"
147+
export POLYCENTRIC_MODERATION_IDENTITY="$MODERATOR_IDENTITY"
148+
# --no-deps avoids pulling in the `scraper` dependency, which requires
149+
# NET_ADMIN for its nftables egress firewall and cannot start in CI's
150+
# Docker-in-Docker environment.
151+
docker compose up -d --no-deps --build --wait server
152+
153+
# Resolve the server container's IP on the compose network and use it
154+
# directly, bypassing Docker embedded DNS (which can be flaky when the job
155+
# container is connected to a compose network via `docker network connect`
156+
# in a Docker-in-Docker environment).
157+
SERVER_HOST=server
158+
SERVER_IP=$(docker inspect -f '{{(index .NetworkSettings.Networks "'${NETWORK}'").IPAddress}}' polycentric-server-1 2>/dev/null)
159+
if [ -n "$SERVER_IP" ]; then
160+
SERVER_HOST=$SERVER_IP
161+
export POLYCENTRIC_TEST_SERVER="http://${SERVER_IP}:3000"
162+
echo " server IP: ${SERVER_IP}"
163+
else
164+
export POLYCENTRIC_TEST_SERVER="${POLYCENTRIC_TEST_SERVER:-http://localhost:3000}"
165+
echo " (no server IP found; using ${POLYCENTRIC_TEST_SERVER})"
166+
fi
167+
168+
echo " waiting for server (port ${SERVER_HOST}:3000)…"
169+
for i in $(seq 1 60); do
170+
if (exec 3<>"/dev/tcp/${SERVER_HOST}/3000") 2>/dev/null; then
171+
exec 3>&- 3<&-
172+
echo " server ready (after ${i}s)"
173+
break
174+
fi
175+
if [ "$i" -eq 60 ]; then
176+
echo "ERROR: server did not start within 60 seconds"
177+
exit 1
178+
fi
179+
sleep 1
180+
done
181+
182+
echo "==> Applying migrations via docker compose exec…"
183+
docker compose exec -T server /app/migration up
184+
echo " migrations applied"
185+
else
186+
echo "==> Applying migrations…"
187+
(
188+
cd services/server/migration
189+
DATABASE_URL="${DATABASE_URL:-postgres://postgres:testing@localhost:5432}" \
190+
cargo run -- fresh 2>&1
191+
)
192+
echo " migrations applied"
193+
194+
echo "==> Starting server…"
195+
export POLYCENTRIC_MODERATION_IDENTITY="$MODERATOR_IDENTITY"
196+
export RUST_LOG="${RUST_LOG:-info}"
197+
export DATABASE_URL="${DATABASE_URL:-postgres://postgres:testing@localhost:5432}"
198+
export CONTENT_BLOB_OS_BUCKET="${CONTENT_BLOB_OS_BUCKET:-polycentric-blobs}"
199+
export CONTENT_BLOB_OS_ENDPOINT="${CONTENT_BLOB_OS_ENDPOINT:-http://localhost:9000}"
200+
export CONTENT_BLOB_OS_FORCE_PATH_STYLE="${CONTENT_BLOB_OS_FORCE_PATH_STYLE:-true}"
201+
export CONTENT_BLOB_OS_ACCESS_KEY="${CONTENT_BLOB_OS_ACCESS_KEY:-rustfsadmin}"
202+
export CONTENT_BLOB_OS_SECRET_KEY="${CONTENT_BLOB_OS_SECRET_KEY:-rustfsadmin}"
203+
# Kafka is reached on the EXTERNAL listener for local connections.
204+
export POLYCENTRIC_KAFKA_BROKERS="${POLYCENTRIC_KAFKA_BROKERS:-localhost:9092}"
205+
# The test crate reads this to know where to reach the server.
206+
export POLYCENTRIC_TEST_SERVER="${POLYCENTRIC_TEST_SERVER:-http://localhost:3000}"
207+
208+
cargo run -p server &
209+
SERVER_PID=$!
210+
echo " server PID $SERVER_PID"
211+
212+
# Wait for the HTTP health endpoint.
213+
echo " waiting for server (compile + boot can take several minutes)…"
214+
for i in $(seq 1 300); do
215+
if (exec 3<>"/dev/tcp/localhost/3000") 2>/dev/null; then
216+
exec 3>&- 3<&-
217+
echo " server ready (after ${i}s)"
218+
break
219+
fi
220+
if [ "$i" -eq 300 ]; then
221+
echo "ERROR: server did not start within 300 seconds"
222+
exit 1
223+
fi
224+
sleep 1
225+
done
226+
fi
227+
228+
# ---------------------------------------------------------------------------
229+
# 3. Run the integration test suite
230+
# ---------------------------------------------------------------------------
231+
echo ""
232+
echo "==> Running integration tests…"
233+
cargo test -p integration-tests 2>&1
234+
echo ""
235+
echo "==> Tests completed"

‎.gitlab/ci/test_moderation.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@
99
.moderation-integration-workflow:
1010
image: $RUST_DIND_IMAGE
1111
stage: test
12-
# Enforce serial execution of moderation integration tests using a `resource_group`, so that the
13-
# Docker stack of one CI runner doing the test doesn't interfere with that of the other.
14-
resource_group: moderation-integration
12+
# Enforce serial execution of integration tests that share the same Docker
13+
# Compose project (postgres, rustfs, kafka, server) — the two integration
14+
# pipelines must never run concurrently or they tear down each other's stack.
15+
resource_group: docker-compose-integration
1516
rules:
1617
- if: $CI_COMMIT_TAG =~ /^moderation-/
1718
when: always

‎.gitlab/ci/test_server.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
########################################################################
2+
# server — integration tests
3+
#
4+
# End-to-end tests against a full server stack (PostgreSQL, RustFS,
5+
# Kafka). Uses the same deterministic test moderator identity as the
6+
# local test script.
7+
########################################################################
8+
9+
.server-integration-workflow:
10+
image: $RUST_DIND_IMAGE
11+
stage: test
12+
# Serial execution: the Docker stack of one runner must not interfere
13+
# with another across integration test jobs (they share the same Compose
14+
# project name).
15+
resource_group: docker-compose-integration
16+
rules:
17+
- if: $CI_COMMIT_TAG =~ /^server-/
18+
when: always
19+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
20+
changes:
21+
- Cargo.toml
22+
- Cargo.lock
23+
- services/server/**/*
24+
- services/common/**/*
25+
- compose.yml
26+
- .gitlab/ci/test_server.yml
27+
- .gitlab/ci/scripts/integration-server.sh
28+
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
29+
when: never
30+
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
31+
changes:
32+
- Cargo.toml
33+
- Cargo.lock
34+
- services/server/**/*
35+
- services/common/**/*
36+
- compose.yml
37+
- .gitlab/ci/test_server.yml
38+
- .gitlab/ci/scripts/integration-server.sh
39+
variables:
40+
RUNNER_SCRIPT_TIMEOUT: 30m
41+
42+
server-integration:
43+
extends: .server-integration-workflow
44+
script:
45+
- .gitlab/ci/scripts/integration-server.sh --ci

‎compose.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ services:
2727
- CONTENT_BLOB_OS_ENDPOINT=http://rustfs:9000
2828
- CONTENT_BLOB_OS_FORCE_PATH_STYLE=true
2929
- CONTENT_BLOB_OS_ACCESS_KEY=rustfsadmin
30+
- POLYCENTRIC_MODERATION_IDENTITY=${POLYCENTRIC_MODERATION_IDENTITY:-}
3031
- CONTENT_BLOB_OS_SECRET_KEY=rustfsadmin
3132
# In-network listener: the default (localhost:9092) points at the server
3233
# container itself, making Kafka unreachable

0 commit comments

Comments
 (0)