Skip to content

Commit 7f0502e

Browse files
kimShubham8287
andauthored
Replication Smoketest Cleanups (#2675)
Co-authored-by: Shubham Mishra <shubham@clockworklabs.io> Co-authored-by: Shubham Mishra <shivam828787@gmail.com>
1 parent 59faab8 commit 7f0502e

6 files changed

Lines changed: 555 additions & 52 deletions

File tree

.github/workflows/ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ jobs:
6060
with: { python-version: '3.12' }
6161
if: runner.os == 'Windows'
6262
- name: Run smoketests
63-
# Note: clear_database only works in private
64-
run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database
63+
# Note: clear_database and replication only work in private
64+
run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication
6565
- name: Stop containers (Linux)
6666
if: always() && runner.os == 'Linux'
6767
run: docker compose down

smoketests/__init__.py

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import re
77
import shutil
88
import string
9-
import string
109
import subprocess
1110
import sys
1211
import tempfile
@@ -38,6 +37,9 @@
3837
# and a dotnet installation is detected
3938
HAVE_DOTNET = False
4039

40+
# default value can be overriden by `--compose-file` flag
41+
COMPOSE_FILE = "./docker-compose.yml"
42+
4143
# we need to late-bind the output stream to allow unittests to capture stdout/stderr.
4244
class CapturableHandler(logging.StreamHandler):
4345

@@ -113,7 +115,7 @@ def run_cmd(*args, capture_stderr=True, check=True, full_output=False, cmd_name=
113115

114116
needs_close = False
115117
if not capture_stderr:
116-
logging.debug(f"--- stderr ---")
118+
logging.debug("--- stderr ---")
117119
needs_close = True
118120

119121
output = subprocess.run(
@@ -172,6 +174,12 @@ def call(self, reducer, *args, anon=False):
172174
anon = ["--anonymous"] if anon else []
173175
self.spacetime("call", *anon, "--", self.database_identity, reducer, *map(json.dumps, args))
174176

177+
178+
def sql(self, sql):
179+
self._check_published()
180+
anon = ["--anonymous"]
181+
return self.spacetime("sql", *anon, "--", self.database_identity, sql)
182+
175183
def logs(self, n):
176184
return [log["message"] for log in self.log_records(n)]
177185

@@ -181,6 +189,7 @@ def log_records(self, n):
181189
return list(map(json.loads, logs.splitlines()))
182190

183191
def publish_module(self, domain=None, *, clear=True, capture_stderr=True):
192+
print("publishing module", self.publish_module)
184193
publish_output = self.spacetime(
185194
"publish",
186195
*[domain] if domain is not None else [],
@@ -210,7 +219,7 @@ def subscribe(self, *queries, n):
210219
self._check_published()
211220
assert isinstance(n, int)
212221

213-
args = [SPACETIME_BIN, "--config-path", str(self.config_path),"subscribe", self.database_identity, "-t", "60", "-n", str(n), "--print-initial-update", "--", *queries]
222+
args = [SPACETIME_BIN, "--config-path", str(self.config_path),"subscribe", self.database_identity, "-t", "600", "-n", str(n), "--print-initial-update", "--", *queries]
214223
fake_args = ["spacetime", *args[1:]]
215224
log_cmd(fake_args)
216225

@@ -294,12 +303,12 @@ def tearDown(self):
294303

295304
@classmethod
296305
def tearDownClass(cls):
297-
if hasattr(cls, "database_identity"):
298-
try:
299-
# TODO: save the credentials in publish_module()
300-
cls.spacetime("delete", cls.database_identity)
301-
except Exception:
302-
pass
306+
if hasattr(cls, "database_identity"):
307+
try:
308+
# TODO: save the credentials in publish_module()
309+
cls.spacetime("delete", cls.database_identity)
310+
except Exception:
311+
pass
303312

304313
if sys.version_info < (3, 11):
305314
# polyfill; python 3.11 defines this classmethod on TestCase

smoketests/__main__.py

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,13 @@
1111
import smoketests
1212
import sys
1313
import logging
14+
import itertools
1415

1516
def check_docker():
1617
docker_ps = smoketests.run_cmd("docker", "ps", "--format=json")
1718
docker_ps = (json.loads(line) for line in docker_ps.splitlines())
1819
for docker_container in docker_ps:
19-
if "node" in docker_container["Image"]:
20+
if "node" in docker_container["Image"] or "spacetime" in docker_container["Image"]:
2021
return docker_container["Names"]
2122
else:
2223
print("Docker container not found, is SpacetimeDB running?")
@@ -51,13 +52,16 @@ def loadTestsFromName(self, name, module=None):
5152
def _convert_select_pattern(pattern):
5253
return f'*{pattern}*' if '*' not in pattern else pattern
5354

55+
5456
TESTPREFIX = "smoketests.tests."
5557
def main():
5658
tests = [fname.removesuffix(".py") for fname in os.listdir(TEST_DIR / "tests") if fname.endswith(".py") and fname != "__init__.py"]
5759

5860
parser = argparse.ArgumentParser()
5961
parser.add_argument("test", nargs="*", default=tests)
6062
parser.add_argument("--docker", action="store_true")
63+
parser.add_argument("--compose-file")
64+
parser.add_argument("--no-docker-logs", action="store_true")
6165
parser.add_argument("--skip-dotnet", action="store_true", help="ignore tests which require dotnet")
6266
parser.add_argument("--show-all-output", action="store_true", help="show all stdout/stderr from the tests as they're running")
6367
parser.add_argument("--parallel", action="store_true", help="run test classes in parallel")
@@ -67,6 +71,7 @@ def main():
6771
help='Only run tests which match the given substring')
6872
parser.add_argument("-x", dest="exclude", nargs="*", default=[])
6973
parser.add_argument("--no-build-cli", action="store_true", help="don't cargo build the cli")
74+
parser.add_argument("--list", action="store_true", help="list the tests that would be run, but don't run them")
7075
args = parser.parse_args()
7176

7277
if not args.no_build_cli:
@@ -94,9 +99,15 @@ def main():
9499
build_template_target()
95100

96101
if args.docker:
97-
docker_container = check_docker()
98102
# have docker logs print concurrently with the test output
99-
subprocess.Popen(["docker", "logs", "-f", docker_container])
103+
if args.compose_file:
104+
smoketests.COMPOSE_FILE = args.compose_file
105+
if not args.no_docker_logs:
106+
if args.compose_file:
107+
subprocess.Popen(["docker", "compose", "-f", args.compose_file, "logs", "-f"])
108+
else:
109+
docker_container = check_docker()
110+
subprocess.Popen(["docker", "logs", "-f", docker_container])
100111
smoketests.HAVE_DOCKER = True
101112

102113
smoketests.new_identity(TEST_DIR / 'config.toml')
@@ -116,6 +127,12 @@ def main():
116127
loader.testNamePatterns = args.testNamePatterns
117128

118129
tests = loader.loadTestsFromNames(testlist)
130+
if args.list:
131+
print("Selected tests:\n")
132+
for test in itertools.chain(*itertools.chain(*tests)):
133+
print(f"{test}")
134+
exit(0)
135+
119136
buffer = not args.show_all_output
120137
verbosity = 2
121138

smoketests/docker.py

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
from dataclasses import dataclass
2+
import os
3+
import subprocess
4+
import time
5+
from typing import List, Optional
6+
from urllib.request import urlopen
7+
from . import COMPOSE_FILE
8+
import json
9+
10+
def restart_docker():
11+
docker = DockerManager(COMPOSE_FILE)
12+
# Restart all containers.
13+
docker.compose("restart")
14+
# Ensure all nodes are reachable from outside.
15+
containers = docker.list_containers()
16+
for container in containers:
17+
info = json.loads(docker._execute_command("docker", "inspect", container.name))
18+
try:
19+
port = info[0]['NetworkSettings']['Ports']['80/tcp'][0]['HostPort']
20+
except KeyError:
21+
continue
22+
ping("127.0.0.1:{}".format(port))
23+
# TODO: ping endpoint needs to wait for database startup & leader election
24+
time.sleep(2)
25+
26+
def ping(host):
27+
tries = 0
28+
while tries < 10:
29+
tries += 1
30+
try:
31+
print(f"Ping Server at {host}")
32+
urlopen(f"http://{host}/v1/ping")
33+
print(f"Server up after {tries} tries")
34+
break
35+
except Exception:
36+
print("Server down")
37+
time.sleep(3)
38+
else:
39+
raise Exception(f"Server at {host} not responding")
40+
41+
@dataclass
42+
class DockerContainer:
43+
"""Represents a Docker container with its basic properties."""
44+
id: str
45+
name: str
46+
47+
class DockerManager:
48+
"""Manages all Docker and Docker Compose operations."""
49+
50+
def __init__(self, compose_file: str, **config):
51+
self.compose_file = compose_file
52+
self.network_name = config.get('network_name') or \
53+
os.getenv('DOCKER_NETWORK_NAME', 'private_spacetime_cloud')
54+
self.control_db_container = config.get('control_db_container') or \
55+
os.getenv('CONTROL_DB_CONTAINER', 'node')
56+
self.spacetime_cli_bin = config.get('spacetime_cli_bin') or \
57+
os.getenv('SPACETIME_CLI_BIN', 'spacetimedb-cloud')
58+
59+
def _execute_command(self, *args: str) -> str:
60+
"""Execute a Docker command and return its output."""
61+
try:
62+
result = subprocess.run(
63+
args,
64+
capture_output=True,
65+
text=True,
66+
check=True
67+
)
68+
return result.stdout.strip()
69+
except subprocess.CalledProcessError as e:
70+
print(f"Command failed: {e.stderr}")
71+
raise
72+
except Exception as e:
73+
print(f"Unexpected error: {str(e)}")
74+
raise
75+
76+
def compose(self, *args: str) -> str:
77+
"""Execute a docker-compose command."""
78+
return self._execute_command("docker", "compose", "-f", self.compose_file, *args)
79+
80+
def list_containers(self) -> List[DockerContainer]:
81+
"""List all containers and return as DockerContainer objects."""
82+
output = self.compose("ps", "-a", "--format", "{{.ID}} {{.Name}}")
83+
containers = []
84+
for line in output.splitlines():
85+
if line.strip():
86+
container_id, name = line.split(maxsplit=1)
87+
containers.append(DockerContainer(id=container_id, name=name))
88+
return containers
89+
90+
def get_container_by_name(self, name: str) -> Optional[DockerContainer]:
91+
"""Find a container by name pattern."""
92+
return next(
93+
(c for c in self.list_containers() if name in c.name),
94+
None
95+
)
96+
97+
def kill_container(self, container_id: str):
98+
"""Kill a container by ID."""
99+
print(f"Killing container {container_id}")
100+
self._execute_command("docker", "kill", container_id)
101+
102+
def start_container(self, container_id: str):
103+
"""Start a container by ID."""
104+
print(f"Starting container {container_id}")
105+
self._execute_command("docker", "start", container_id)
106+
107+
def disconnect_container(self, container_id: str):
108+
"""Disconnect a container from the network."""
109+
print(f"Disconnecting container {container_id}")
110+
self._execute_command(
111+
"docker", "network", "disconnect",
112+
self.network_name, container_id
113+
)
114+
print(f"Disconnected container {container_id}")
115+
116+
def connect_container(self, container_id: str):
117+
"""Connect a container to the network."""
118+
print(f"Connecting container {container_id}")
119+
self._execute_command(
120+
"docker", "network", "connect",
121+
self.network_name, container_id
122+
)
123+
print(f"Connected container {container_id}")
124+
125+
def generate_root_token(self) -> str:
126+
"""Generate a root token using spacetimedb-cloud."""
127+
return self.compose(
128+
"exec", self.control_db_container, self.spacetime_cli_bin, "token", "gen",
129+
"--subject=placeholder-node-id",
130+
"--jwt-priv-key", "/etc/spacetimedb/keys/id_ecdsa").split('|')[1]

0 commit comments

Comments
 (0)