Skip to content

Commit a29906d

Browse files
deacon-mpdeacon-mp
authored
fix: register SIGTERM handler in run_tasks() to save state on service shutdown (#3018) (#3277)
* fix: register SIGTERM handler in run_tasks() to ensure teardown on service shutdown When Caldera runs as a systemd service (or backgrounded via & / nohup), shutdown sends SIGTERM rather than SIGINT/KeyboardInterrupt. Without a handler, the existing 'except KeyboardInterrupt' teardown block is never reached, so operations and other in-memory state are not saved to disk (issue #3018). Register a SIGTERM handler at the start of run_tasks() that converts the signal into KeyboardInterrupt, reusing the established teardown path without duplicating logic. Tests added to verify structure (AST) and runtime behaviour. * style: remove unused imports in test_server_sigterm.py * fix: wrap full startup in try/except so teardown runs on SIGTERM during startup; add AST test
1 parent 973ed61 commit a29906d

2 files changed

Lines changed: 196 additions & 8 deletions

File tree

‎server.py‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import asyncio
33
import logging
44
import os
5+
import signal
56
from rich.console import Console
67
from rich.logging import RichHandler
78
from rich.theme import Theme
@@ -74,6 +75,18 @@ async def start_server():
7475

7576

7677
def run_tasks(services, run_vue_server=False):
78+
def _handle_sigterm(*args):
79+
"""Convert SIGTERM into KeyboardInterrupt to reuse the existing teardown path.
80+
81+
When Caldera runs as a systemd service (or with ``& disown`` / ``nohup``),
82+
the process receives SIGTERM on shutdown rather than SIGINT/KeyboardInterrupt.
83+
Without this handler the teardown/save logic is never called, so operations
84+
and other in-memory state are lost (issue #3018).
85+
"""
86+
raise KeyboardInterrupt
87+
88+
signal.signal(signal.SIGTERM, _handle_sigterm)
89+
7790
loop = asyncio.new_event_loop()
7891
# The event loop is set here, before any async work begins. Services
7992
# (AppService, DataService, etc.) are instantiated in __main__ prior to
@@ -123,14 +136,13 @@ def run_tasks(services, run_vue_server=False):
123136
)
124137
if run_vue_server:
125138
loop.run_until_complete(start_vue_dev_server())
126-
try:
127-
logging.info("All systems ready.")
128-
print_rich_banner()
129-
loop.run_forever()
130-
except KeyboardInterrupt:
131-
loop.run_until_complete(
132-
services.get("app_svc").teardown(main_config_file=args.environment)
133-
)
139+
logging.info("All systems ready.")
140+
print_rich_banner()
141+
loop.run_forever()
142+
except KeyboardInterrupt:
143+
loop.run_until_complete(
144+
services.get("app_svc").teardown(main_config_file=args.environment)
145+
)
134146
finally:
135147
# Cancel all pending tasks before shutdown to avoid resource leaks
136148
# and "Task was destroyed but it is pending!" warnings.

‎tests/test_server_sigterm.py‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
"""Tests for SIGTERM handling in server.py (issue #3018).
2+
3+
When Caldera runs as a systemd service, shutdown sends SIGTERM rather than
4+
SIGINT/KeyboardInterrupt. Without a handler, teardown() is never called and
5+
in-memory state (operations, etc.) is lost. The fix registers a SIGTERM
6+
handler that converts the signal into KeyboardInterrupt so the existing
7+
teardown path is reused.
8+
"""
9+
import ast
10+
import os
11+
import signal
12+
import unittest
13+
14+
15+
# ---------------------------------------------------------------------------
16+
# AST-level structural check
17+
# ---------------------------------------------------------------------------
18+
19+
class TestSigtermHandlerStructure(unittest.TestCase):
20+
"""Verify, without importing server.py, that the SIGTERM handler is
21+
registered inside run_tasks() using pure AST inspection."""
22+
23+
def _parse_server(self):
24+
server_path = os.path.join(
25+
os.path.dirname(__file__), '..', 'server.py'
26+
)
27+
with open(os.path.normpath(server_path)) as fh:
28+
return ast.parse(fh.read())
29+
30+
def test_signal_module_imported(self):
31+
tree = self._parse_server()
32+
imports = [
33+
node for node in ast.walk(tree)
34+
if isinstance(node, ast.Import)
35+
]
36+
signal_imported = any(
37+
alias.name == 'signal'
38+
for imp in imports
39+
for alias in imp.names
40+
)
41+
self.assertTrue(signal_imported, "'import signal' not found in server.py")
42+
43+
def _get_run_tasks_body(self):
44+
tree = self._parse_server()
45+
for node in ast.walk(tree):
46+
if isinstance(node, ast.FunctionDef) and node.name == 'run_tasks':
47+
return node
48+
return None
49+
50+
def test_sigterm_registered_in_run_tasks(self):
51+
run_tasks = self._get_run_tasks_body()
52+
self.assertIsNotNone(run_tasks, "run_tasks() function not found in server.py")
53+
54+
# Look for signal.signal(signal.SIGTERM, ...) call anywhere inside run_tasks
55+
sigterm_calls = []
56+
for node in ast.walk(run_tasks):
57+
if not isinstance(node, ast.Call):
58+
continue
59+
func = node.func
60+
if not (isinstance(func, ast.Attribute) and func.attr == 'signal'):
61+
continue
62+
if len(node.args) < 1:
63+
continue
64+
first_arg = node.args[0]
65+
if isinstance(first_arg, ast.Attribute) and first_arg.attr == 'SIGTERM':
66+
sigterm_calls.append(node)
67+
68+
self.assertTrue(
69+
len(sigterm_calls) >= 1,
70+
"signal.signal(signal.SIGTERM, ...) not found inside run_tasks() in server.py"
71+
)
72+
73+
def test_sigterm_handler_raises_keyboard_interrupt(self):
74+
"""The SIGTERM handler function body must raise KeyboardInterrupt."""
75+
run_tasks = self._get_run_tasks_body()
76+
self.assertIsNotNone(run_tasks)
77+
78+
# Find signal.signal(signal.SIGTERM, handler_name) and then verify the
79+
# handler's body raises KeyboardInterrupt.
80+
handler_name = None
81+
for node in ast.walk(run_tasks):
82+
if not isinstance(node, ast.Call):
83+
continue
84+
func = node.func
85+
if not (isinstance(func, ast.Attribute) and func.attr == 'signal'):
86+
continue
87+
if len(node.args) < 2:
88+
continue
89+
first_arg = node.args[0]
90+
if isinstance(first_arg, ast.Attribute) and first_arg.attr == 'SIGTERM':
91+
second_arg = node.args[1]
92+
if isinstance(second_arg, ast.Name):
93+
handler_name = second_arg.id
94+
break
95+
96+
self.assertIsNotNone(handler_name, "Could not determine SIGTERM handler name")
97+
98+
# Locate the handler function definition inside run_tasks
99+
for node in ast.walk(run_tasks):
100+
if isinstance(node, ast.FunctionDef) and node.name == handler_name:
101+
raises = [
102+
n for n in ast.walk(node)
103+
if isinstance(n, ast.Raise)
104+
and n.exc is not None
105+
and (
106+
(isinstance(n.exc, ast.Call) and
107+
isinstance(n.exc.func, ast.Name) and
108+
n.exc.func.id == 'KeyboardInterrupt')
109+
or
110+
(isinstance(n.exc, ast.Name) and n.exc.id == 'KeyboardInterrupt')
111+
)
112+
]
113+
self.assertTrue(
114+
len(raises) >= 1,
115+
f"Handler '{handler_name}' does not raise KeyboardInterrupt"
116+
)
117+
return
118+
119+
self.fail(f"Handler function '{handler_name}' not found inside run_tasks()")
120+
121+
def test_startup_wrapped_in_try_except(self):
122+
"""run_tasks() must wrap the full startup sequence (not just run_forever) in a
123+
try/except KeyboardInterrupt so that SIGTERM during startup still triggers teardown."""
124+
run_tasks = self._get_run_tasks_body()
125+
self.assertIsNotNone(run_tasks, "run_tasks() not found in server.py")
126+
127+
# The try block must contain restore_state (a startup call) AND run_forever,
128+
# proving the handler covers the whole startup, not just the main loop.
129+
for node in ast.walk(run_tasks):
130+
if not isinstance(node, ast.Try):
131+
continue
132+
calls_in_try = []
133+
for child in ast.walk(node):
134+
if isinstance(child, ast.Call) and isinstance(child.func, ast.Attribute):
135+
calls_in_try.append(child.func.attr)
136+
has_startup = any(c in calls_in_try for c in ('restore_state', 'load_plugins'))
137+
has_run_forever = 'run_forever' in calls_in_try
138+
has_teardown = any(
139+
'teardown' in ast.dump(h)
140+
for h in node.handlers
141+
)
142+
if has_startup and has_run_forever and has_teardown:
143+
return
144+
145+
self.fail(
146+
"run_tasks() try/except must cover both startup calls and run_forever "
147+
"so teardown() is called on SIGTERM at any point during startup"
148+
)
149+
150+
151+
# ---------------------------------------------------------------------------
152+
# Behavioural check: sending SIGTERM to self raises KeyboardInterrupt
153+
# ---------------------------------------------------------------------------
154+
155+
class TestSigtermRaisesKeyboardInterrupt(unittest.TestCase):
156+
"""Install the same handler logic used in server.py and verify that
157+
sending SIGTERM to the current process raises KeyboardInterrupt."""
158+
159+
def setUp(self):
160+
self._original_handler = signal.getsignal(signal.SIGTERM)
161+
162+
def tearDown(self):
163+
signal.signal(signal.SIGTERM, self._original_handler)
164+
165+
def test_sigterm_raises_keyboard_interrupt(self):
166+
def _handle_sigterm(*args):
167+
raise KeyboardInterrupt
168+
169+
signal.signal(signal.SIGTERM, _handle_sigterm)
170+
171+
with self.assertRaises(KeyboardInterrupt):
172+
os.kill(os.getpid(), signal.SIGTERM)
173+
174+
175+
if __name__ == '__main__':
176+
unittest.main()

0 commit comments

Comments
 (0)