Skip to content

Commit e728ea5

Browse files
steventuxsteventux
committed
Rotate logs and backup database on deployed gateways
Sets up a Windows schedule to backup databases and rotate logs every Sunday at 3am.
1 parent 9d3e46f commit e728ea5

3 files changed

Lines changed: 217 additions & 1 deletion

File tree

‎scripts/bash/package_release.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ echo ""
6161

6262
# ── Validate required files ───────────────────────────────────────────────────
6363

64-
REQUIRED_FILES=("src/" "sample_images/" "scripts/python/" "pyproject.toml" "uv.lock" "README.md" "LICENCE.md")
64+
REQUIRED_FILES=("src/" "sample_images/" "scripts/python/" "scripts/powershell/maintenance.ps1" "pyproject.toml" "uv.lock" "README.md" "LICENCE.md")
6565

6666
for item in "${REQUIRED_FILES[@]}"; do
6767
if [[ ! -e "${REPO_ROOT}/${item}" ]]; then

‎scripts/powershell/deploy.ps1‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,39 @@ if (-not $cutoverFailed) {
560560
}
561561
}
562562

563+
# -- Maintenance schedule -------------------------------------------------------------
564+
565+
if (-not $cutoverFailed) {
566+
$maintenanceScriptPath = Join-Path $BaseInstallPath "current\scripts\powershell\maintenance.ps1"
567+
$backupAction = New-ScheduledTaskAction `
568+
-Execute "powershell.exe" `
569+
-Argument "-ExecutionPolicy Bypass -File `"$maintenanceScriptPath`" -Action StopBackupRotate"
570+
571+
$backupTrigger = New-ScheduledTaskTrigger -Weekly -DaysOfWeek Wednesday -At 9:15AM
572+
573+
Register-ScheduledTask `
574+
-TaskName "Gateway-Daily-Maintenance" `
575+
-Action $backupAction `
576+
-Trigger $backupTrigger `
577+
-User "SYSTEM" `
578+
-RunLevel Highest `
579+
-Force
580+
581+
$startAction = New-ScheduledTaskAction `
582+
-Execute "powershell.exe" `
583+
-Argument "-ExecutionPolicy Bypass -File `"$maintenanceScriptPath`" -Action StartServices"
584+
585+
$startTrigger = New-ScheduledTaskTrigger -Weekly -DaysOfWeek Wednesday -At 9:30AM
586+
587+
Register-ScheduledTask `
588+
-TaskName "Gateway-Daily-Service-Start" `
589+
-Action $startAction `
590+
-Trigger $startTrigger `
591+
-User "SYSTEM" `
592+
-RunLevel Highest `
593+
-Force
594+
}
595+
563596
# -- Rollback on Failure ------------------------------------------------------
564597

565598
if ($cutoverFailed) {

‎scripts/powershell/maintenance.ps1‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
param(
2+
[ValidateSet("StopBackupRotate","StartServices")]
3+
[string]$BaseInstallPath = "C:\Program Files\NHS\ManageBreastScreeningGateway",
4+
[string]$Action
5+
)
6+
7+
# -- Set common vars -------------------------------------------------------
8+
9+
$logsDir = Join-Path $BaseInstallPath "logs"
10+
$maintenanceLogFile = Join-Path $logsDir "maintenance.log"
11+
12+
$services = Get-Service |
13+
Where-Object { $_.Name -like "Gateway-*" } |
14+
ForEach-Object {
15+
@{ Name = $_.Name }
16+
}
17+
18+
function Write-Log {
19+
param([string]$Message, [string]$Level = "INFO")
20+
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff"
21+
$logEntry = "[$timestamp] [$Level] $Message"
22+
Add-Content -Path $maintenanceLogFile -Value $logEntry
23+
switch ($Level) {
24+
"ERROR" { Write-Host $logEntry -ForegroundColor Red }
25+
"WARNING" { Write-Host $logEntry -ForegroundColor Yellow }
26+
"SUCCESS" { Write-Host $logEntry -ForegroundColor Green }
27+
default { Write-Host $logEntry -ForegroundColor Gray }
28+
}
29+
}
30+
31+
# -- Service control helpers -------------------------------------------------------
32+
33+
function Stop-AllServices {
34+
param([array]$Services, [int]$TimeoutSeconds)
35+
foreach ($svc in $Services) {
36+
$status = Get-Service -Name $svc.Name -ErrorAction SilentlyContinue
37+
if (-not $status -or $status.Status -eq 'Stopped') { continue }
38+
39+
Write-Log "Stopping $($svc.Name) (timeout: ${TimeoutSeconds}s)..." "INFO"
40+
Stop-Service -Name $svc.Name -Force
41+
42+
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
43+
while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
44+
$current = Get-Service -Name $svc.Name -ErrorAction SilentlyContinue
45+
if (-not $current -or $current.Status -eq 'Stopped') { break }
46+
Start-Sleep -Milliseconds 500
47+
}
48+
$stopwatch.Stop()
49+
50+
$finalStatus = Get-Service -Name $svc.Name -ErrorAction SilentlyContinue
51+
if ($finalStatus -and $finalStatus.Status -ne 'Stopped') {
52+
throw "Service $($svc.Name) did not stop within ${TimeoutSeconds}s (state: $($finalStatus.Status))."
53+
}
54+
Write-Log "$($svc.Name) stopped in $([math]::Round($stopwatch.Elapsed.TotalSeconds, 1))s." "INFO"
55+
}
56+
}
57+
58+
function Start-AllServices {
59+
param(
60+
[array]$Services, [int]$TimeoutSeconds
61+
)
62+
63+
foreach ($svc in $Services) {
64+
Write-Log "Starting $($svc.Name)..." "INFO"
65+
66+
Start-Service -Name $svc.Name
67+
68+
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
69+
while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
70+
$status = Get-Service -Name $svc.Name -ErrorAction SilentlyContinue
71+
if ($status -and $status.Status -eq 'Running') { break }
72+
Start-Sleep -Milliseconds 500
73+
}
74+
75+
if ($status.Status -ne 'Running') {
76+
Write-Log "Failed to start $($svc.Name)" "ERROR"
77+
throw "Service $($svc.Name) did not start within ${TimeoutSeconds}s (state: $($status.Status))."
78+
}
79+
80+
Write-Log "$($svc.Name) started." "SUCCESS"
81+
}
82+
}
83+
84+
# -- Database backup ----------------------------------------------------------
85+
86+
function Invoke-DatabaseBackup {
87+
param(
88+
[string]$BaseInstallPath
89+
)
90+
91+
$batPath = Join-Path $versionDir "backup_database.bat"
92+
93+
if (-not (Test-Path $batPath)) {
94+
$batContent = @(
95+
'@echo off',
96+
"cd /d `"$BaseInstallPath`"",
97+
'set "PYTHONPATH=current\src"',
98+
('"current\.venv\Scripts\python.exe" "-c import scripts.python.database; scripts.python.database.backup_databases()"')
99+
) -join "`r`n"
100+
[System.IO.File]::WriteAllText($batPath, $batContent, [System.Text.Encoding]::ASCII)
101+
}
102+
103+
Write-Log "Starting database backup..." "INFO"
104+
105+
& cmd.exe /c "`"$batPath`""
106+
107+
if ($LASTEXITCODE -ne 0) {
108+
throw "Database backup failed with exit code $LASTEXITCODE"
109+
}
110+
111+
Write-Log "Database backup completed successfully." "SUCCESS"
112+
}
113+
114+
# -- Log Rotation -------------------------------------------------------------
115+
116+
function Rotate-LogFile {
117+
param(
118+
[Parameter(Mandatory)]
119+
[string]$LogFile,
120+
121+
[int]$RetainCount = 5
122+
)
123+
124+
if (-not (Test-Path $LogFile)) {
125+
return
126+
}
127+
128+
# Remove oldest
129+
$oldest = "$LogFile.$RetainCount"
130+
if (Test-Path $oldest) {
131+
Remove-Item $oldest -Force
132+
}
133+
134+
# Shift existing rotations
135+
for ($i = $RetainCount - 1; $i -ge 1; $i--) {
136+
$src = "$LogFile.$i"
137+
$dst = "$LogFile." + ($i + 1)
138+
139+
if (Test-Path $src) {
140+
Move-Item $src $dst -Force
141+
}
142+
}
143+
144+
Move-Item $LogFile "$LogFile.1" -Force
145+
}
146+
147+
function Rotate-ServiceLogs {
148+
param(
149+
[array]$Services,
150+
[string]$LogsDir
151+
)
152+
153+
foreach ($svc in $Services) {
154+
$logFile = Join-Path $LogsDir "$($svc.Name).log"
155+
Rotate-LogFile -LogFile $logFile -RetainCount 5
156+
}
157+
}
158+
159+
# -- Main ----------------------------------------------------------------------
160+
161+
switch ($Action) {
162+
163+
"StopBackupRotate" {
164+
165+
Stop-AllServices `
166+
-Services $services `
167+
-TimeoutSeconds 30
168+
169+
Invoke-DatabaseBackup `
170+
-BaseInstallPath $BaseInstallPath
171+
172+
Rotate-ServiceLogs `
173+
-Services $services `
174+
-LogsDir $logsDir
175+
}
176+
177+
"StartServices" {
178+
179+
Start-AllServices `
180+
-Services $services
181+
-TimeoutSeconds 30
182+
}
183+
}

0 commit comments

Comments
 (0)