-
Notifications
You must be signed in to change notification settings - Fork 4
Expand file tree
/
Copy path_board_generator.py
More file actions
executable file
·123 lines (99 loc) · 4.22 KB
/
Copy path_board_generator.py
File metadata and controls
executable file
·123 lines (99 loc) · 4.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import asyncio
import signal
import sys
from monitor.sdnotifier import SystemdWatchdog
from utils.sentry import init_sentry
init_sentry("droptracker-lootboards")
"""
Lootboard Generator
This process runs as a systemd service to call the actual `board_generator.py` script every 2 minutes.
"""
# Global variables for systemd watchdog
watchdog = None
shutdown_event = asyncio.Event()
# Health check function for systemd watchdog
async def health_check():
"""Simple health check for the lootboard generator"""
try:
# Basic health check - service is running if we get here
return True
except Exception as e:
print(f"Health check failed: {e}")
return False
# Signal handlers for graceful shutdown
def signal_handler(signum, frame):
"""Handle shutdown signals"""
print(f"Received signal {signum}, initiating graceful shutdown...")
shutdown_event.set()
def setup_signal_handlers():
"""Setup signal handlers for graceful shutdown"""
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
async def board_loop():
while not shutdown_event.is_set():
try:
print("Starting board generation process...")
# Use asyncio subprocess to avoid blocking the watchdog
process = await asyncio.create_subprocess_exec(
"/store/droptracker/disc/venv/bin/python", ## Venv location
"-m", "lootboard.board_generator", ## File to execute in subprocess
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd="/store/droptracker/disc" ## Working directory
)
# Wait for process with timeout, but don't block watchdog
try:
stdout, stderr = await asyncio.wait_for(
process.communicate(),
timeout=600 # 10 minute timeout; normal runs are capped well below this
)
if process.returncode != 0:
print(f"Board generation failed with return code {process.returncode}")
print(f"Error output: {stderr.decode() if stderr else 'No error output'}")
else:
print("Board generation completed successfully")
if stdout:
print(f"Output: {stdout.decode()}")
except asyncio.TimeoutError:
print("Board generation timed out after 10 minutes, terminating process...")
process.terminate()
try:
await asyncio.wait_for(process.wait(), timeout=10)
except asyncio.TimeoutError:
print("Process didn't terminate gracefully, killing...")
process.kill()
await process.wait()
except Exception as e:
print(f"Error in board generation: {e}")
print("Board generation process completed & exited. Sleeping for 2 minutes")
# Sleep with interruption check
for _ in range(120): # 2 minutes = 120 seconds
if shutdown_event.is_set():
break
await asyncio.sleep(1)
async def main():
"""Main function with systemd watchdog integration"""
global watchdog
# Setup signal handlers
setup_signal_handlers()
# Initialize systemd watchdog
watchdog = SystemdWatchdog()
watchdog.set_health_check(health_check)
try:
async with watchdog:
# Notify systemd that we're ready
await watchdog.notify_ready()
print("Systemd watchdog initialized and ready notification sent")
# Start the board generation loop
await board_loop()
print("Lootboard generator shutting down gracefully...")
except KeyboardInterrupt:
print("Received keyboard interrupt")
except Exception as e:
print(f"Fatal error in main: {e}")
raise
finally:
print("Lootboard generator cleanup completed")
if __name__ == "__main__":
asyncio.run(main())