Skip to content

Commit 9950ca2

Browse files
hyperpolymathhyperpolymath
committed
Auto-commit: Sync changes [2026-03-02]
1 parent b1f120a commit 9950ca2

2 files changed

Lines changed: 246 additions & 6 deletions

File tree

‎cloud-mount-watcher.sh‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: PMPL-1.0-or-later
3+
# cloud-mount-watcher.sh - Detects failed rclone cloud mounts and auto-remounts
4+
# Checks: mount presence, FUSE responsiveness, rclone process health
5+
# Runs via systemd timer every 60 seconds
6+
#
7+
# Notification behaviour:
8+
# - Repair SUCCESS: persistent notification (user must dismiss)
9+
# - Repair FAILURE: critical persistent notification, auto-dismissed when recovered
10+
# - Healthy after prior failure: auto-clears the failure notification
11+
12+
set -euo pipefail
13+
14+
readonly LOG_DIR="${HOME}/.local/share/dependability"
15+
readonly LOG_FILE="${LOG_DIR}/cloud-mount-watcher.log"
16+
readonly FAIL_STATE_DIR="${LOG_DIR}/cloud-watcher-failures"
17+
mkdir -p "$LOG_DIR" "$FAIL_STATE_DIR"
18+
19+
log() {
20+
local level="$1"; shift
21+
printf '%s [CLOUD-WATCHER] [%s] %s\n' "$(date -Iseconds)" "$level" "$*" >> "$LOG_FILE"
22+
}
23+
24+
# Send a desktop notification with a stable replace-id so we can update/dismiss it.
25+
# Usage: send_notify urgency replace_tag title body
26+
send_notify() {
27+
local urgency="$1" tag="$2" title="$3" body="$4"
28+
if command -v notify-send &>/dev/null; then
29+
# --hint string:x-canonical-private-synchronous:TAG keeps a single
30+
# replaceable slot per TAG so repeated alerts don't spam. The
31+
# "resident" hint asks the DE to keep the notification visible until
32+
# the user acts on it (supported by GNOME 45+, KDE Plasma 6).
33+
notify-send \
34+
-u "$urgency" \
35+
-h "string:x-canonical-private-synchronous:cloud-watcher-${tag}" \
36+
-h "boolean:resident:true" \
37+
"$title" "$body" 2>/dev/null || true
38+
fi
39+
}
40+
41+
# Close/replace a notification by sending a minimal "resolved" update.
42+
# Usage: clear_notify tag
43+
clear_notify() {
44+
local tag="$1"
45+
if command -v notify-send &>/dev/null; then
46+
# Replace the persistent notification with a transient "resolved" one
47+
# that auto-expires after 5 seconds (5000 ms).
48+
notify-send \
49+
-u normal \
50+
-t 5000 \
51+
-h "string:x-canonical-private-synchronous:cloud-watcher-${tag}" \
52+
"Cloud Mount Watcher" "${tag} recovered — mount is healthy" 2>/dev/null || true
53+
fi
54+
}
55+
56+
# Rotate log if > 2MB
57+
if [[ -f "$LOG_FILE" ]] && (( $(stat -c%s "$LOG_FILE" 2>/dev/null || echo 0) > 2097152 )); then
58+
mv "$LOG_FILE" "${LOG_FILE}.old"
59+
log "INFO" "Log rotated"
60+
fi
61+
62+
# Cloud mount definitions: remote_name:mount_point:service_name
63+
declare -a MOUNTS=(
64+
"gdrive:/var/home/hyper/Cloud/GoogleDrive:rclone-gdrive.service"
65+
"onedrive:/var/home/hyper/Cloud/OneDrive:rclone-onedrive.service"
66+
"dropbox:/var/home/hyper/Cloud/Dropbox:rclone-dropbox.service"
67+
)
68+
69+
repaired=0
70+
checked=0
71+
72+
for entry in "${MOUNTS[@]}"; do
73+
IFS=':' read -r remote mount_point service <<< "$entry"
74+
checked=$((checked + 1))
75+
needs_repair=false
76+
reason=""
77+
78+
# Check 1: Is the service supposed to be enabled?
79+
if ! systemctl --user is-enabled "$service" &>/dev/null; then
80+
# Service isn't enabled, skip it (user doesn't want it)
81+
continue
82+
fi
83+
84+
# Check 2: Is it mounted at all?
85+
if ! mountpoint -q "$mount_point" 2>/dev/null; then
86+
needs_repair=true
87+
reason="not mounted"
88+
fi
89+
90+
# Check 3: If mounted, is the FUSE mount responsive? (timeout after 5s)
91+
if [[ "$needs_repair" == "false" ]]; then
92+
if ! timeout 5 ls "$mount_point" &>/dev/null; then
93+
needs_repair=true
94+
reason="mount unresponsive (stale FUSE)"
95+
fi
96+
fi
97+
98+
# Check 4: Is the rclone process actually running for this mount?
99+
if [[ "$needs_repair" == "false" ]]; then
100+
if ! systemctl --user is-active "$service" &>/dev/null; then
101+
needs_repair=true
102+
reason="service not active"
103+
fi
104+
fi
105+
106+
# BACKOFF & LOAD PROTECTION
107+
backoff_file="${FAIL_STATE_DIR}/${remote}.backoff"
108+
109+
if [[ "$needs_repair" == "true" ]]; then
110+
# Check system load - don't add fuel to the fire
111+
load=$(awk '{print int($1)}' /proc/loadavg)
112+
if (( load > 4 )); then
113+
log "WARN" "System load high (${load}), deferring repair for ${remote}"
114+
continue
115+
fi
116+
117+
# Check backoff counter
118+
failures=$(cat "$backoff_file" 2>/dev/null || echo 0)
119+
failures=$((failures + 1))
120+
echo "$failures" > "$backoff_file"
121+
122+
# Exponential backoff: 1, 2, 4, 8, 16...
123+
# If failures=3, wait until cycle is multiple of 4 (2^2)
124+
# If failures=5, wait until cycle is multiple of 16 (2^4)
125+
backoff_limit=6 # Max backoff 2^6 = 64 cycles (~1 hour)
126+
current_backoff=$(( failures > backoff_limit ? backoff_limit : failures ))
127+
wait_cycles=$(( 2 ** (current_backoff - 1) ))
128+
129+
# We use the global count file as a tick counter
130+
global_tick=$(cat "${LOG_DIR}/.cloud-watcher-count" 2>/dev/null || echo 0)
131+
132+
# If we are NOT on a retry tick, skip
133+
if (( global_tick % wait_cycles != 0 )); then
134+
log "INFO" "${remote} failed, backing off (attempt ${failures}, waiting for next window)"
135+
continue
136+
fi
137+
138+
log "ALERT" "${remote} mount failed: ${reason} — attempting repair (attempt ${failures})"
139+
140+
# Mark this mount as having an outstanding failure
141+
echo "$reason" > "${FAIL_STATE_DIR}/${remote}"
142+
143+
# Send persistent failure notification (stays until dismissed or cleared)
144+
send_notify "critical" "$remote" \
145+
"Cloud Mount Watcher" \
146+
"${remote} mount failed: ${reason} — attempting repair..."
147+
148+
# Step 1: Clean up stale FUSE mount if present
149+
if mountpoint -q "$mount_point" 2>/dev/null; then
150+
fusermount -u "$mount_point" 2>/dev/null || true
151+
sleep 1
152+
fi
153+
154+
# Step 2: Force-unmount if still stuck
155+
if mountpoint -q "$mount_point" 2>/dev/null; then
156+
fusermount -uz "$mount_point" 2>/dev/null || true
157+
sleep 1
158+
fi
159+
160+
# Step 3: Reset failed state and restart the service
161+
systemctl --user reset-failed "$service" 2>/dev/null || true
162+
if systemctl --user restart "$service" 2>/dev/null; then
163+
# Wait for the mount to come up (rclone has a 5s sleep + mount time)
164+
sleep 8
165+
if mountpoint -q "$mount_point" 2>/dev/null; then
166+
log "INFO" "${remote} mount repaired successfully"
167+
repaired=$((repaired + 1))
168+
169+
# Clear the failure state
170+
rm -f "${FAIL_STATE_DIR}/${remote}"
171+
172+
# Persistent success notification (user must dismiss)
173+
send_notify "normal" "$remote" \
174+
"Cloud Mount Watcher" \
175+
"${remote} mount REPAIRED successfully (was: ${reason})"
176+
else
177+
log "CRIT" "${remote} mount repair FAILED — service restarted but mount not present"
178+
179+
# Update the failure notification to show repair failed
180+
send_notify "critical" "$remote" \
181+
"Cloud Mount Watcher" \
182+
"${remote} mount repair FAILED — will retry next cycle (was: ${reason})"
183+
fi
184+
else
185+
log "CRIT" "${remote} mount repair FAILED — could not restart ${service}"
186+
187+
send_notify "critical" "$remote" \
188+
"Cloud Mount Watcher" \
189+
"${remote} repair FAILED — could not restart ${service}"
190+
fi
191+
else
192+
# Mount is healthy — if there was a prior failure, clear its notification
193+
if [[ -f "${FAIL_STATE_DIR}/${remote}" ]]; then
194+
log "INFO" "${remote} recovered from previous failure"
195+
rm -f "${FAIL_STATE_DIR}/${remote}"
196+
rm -f "${FAIL_STATE_DIR}/${remote}.backoff" # Reset backoff
197+
clear_notify "$remote"
198+
fi
199+
fi
200+
done
201+
202+
if (( repaired > 0 )); then
203+
log "INFO" "Watcher check complete: ${repaired}/${checked} mount(s) repaired"
204+
elif (( checked > 0 )); then
205+
# Only log periodic health confirmations every ~10 minutes (every 10th check)
206+
count_file="${LOG_DIR}/.cloud-watcher-count"
207+
count=$(cat "$count_file" 2>/dev/null || echo 0)
208+
count=$((count + 1))
209+
echo "$count" > "$count_file"
210+
if (( count % 10 == 0 )); then
211+
log "INFO" "All ${checked} cloud mount(s) healthy"
212+
fi
213+
fi

‎sync-repos.sh‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,45 @@
11
#!/bin/bash
2-
# SPDX-License-Identifier: AGPL-3.0-or-later
3-
# Sync all repos in ~/repos
2+
# SPDX-License-Identifier: PMPL-1.0-or-later
3+
# Sync all git repos in the canonical hyperpolymath repos directory.
4+
# Fetches all remotes, then attempts a fast-forward pull.
5+
# Reports errors per-repo without stopping the batch.
46

5-
cd ~/repos || exit 1
7+
REPO_DIR="$HOME/Documents/hyperpolymath-repos"
8+
9+
cd "$REPO_DIR" || { echo "ERROR: Cannot cd to $REPO_DIR"; exit 1; }
10+
11+
ok=0
12+
warn=0
13+
fail=0
614

715
for repo in */; do
816
if [ -d "$repo/.git" ]; then
917
echo -n "$repo: "
1018
cd "$repo"
11-
if git fetch --all -q 2>/dev/null && git pull -q 2>/dev/null; then
12-
echo "✓"
19+
20+
# Fetch all remotes, capture errors but don't abort
21+
fetch_err=$(git fetch --all -q 2>&1)
22+
fetch_rc=$?
23+
24+
# Attempt pull (fast-forward only to avoid surprise merges)
25+
pull_err=$(git pull --ff-only -q 2>&1)
26+
pull_rc=$?
27+
28+
if [ $fetch_rc -eq 0 ] && [ $pull_rc -eq 0 ]; then
29+
echo "OK"
30+
((ok++))
31+
elif [ $pull_rc -ne 0 ]; then
32+
echo "PULL-FAIL: $pull_err"
33+
((warn++))
1334
else
14-
echo "⚠ (has local changes or conflicts)"
35+
echo "FETCH-FAIL: $fetch_err"
36+
((fail++))
1537
fi
38+
1639
cd ..
1740
fi
1841
done
42+
43+
echo ""
44+
echo "--- Summary ---"
45+
echo "OK: $ok Warnings: $warn Failures: $fail"

0 commit comments

Comments
 (0)