Skip to content

Commit 4ba00fa

Browse files
NetworkCatsclaude
andcommitted
feat: poll MMDB every 6 hours to mitigate upstream release drift
The daily scheduled update at 01:20 UTC was systematically lagging the upstream mergedipdb release by 24h. The upstream cron nominally fires at 01:00 UTC but GitHub Actions schedule latency shifts the actual publish time anywhere from ~01:05 to ~06:00+ UTC, so any fixed single-daily time misses the current release on delayed days. Introduce DB_UPDATE_INTERVAL_HOURS (default 6, must divide 24 evenly), shift the anchor DB_UPDATE_TIME_UTC default to 00:20, and replace secs_until_next with secs_until_next_tick that schedules at anchor + N*interval. Also set RUST_LOG=rustyip=info,warn in the Ansible env template so the scheduler and "database updated successfully (N bytes)" info logs are actually visible in docker logs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent a17b105 commit 4ba00fa

6 files changed

Lines changed: 172 additions & 37 deletions

File tree

.env.example

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,14 @@ DB_PATH=data/Merged-IP.mmdb
77
# URL to download the MMDB database from
88
DB_UPDATE_URL=https://github.com/NetworkCats/Merged-IP-Data/releases/latest/download/Merged-IP.mmdb
99

10-
# Daily UTC time to update the database (HH:MM format)
11-
# Merged IP Data releases at 01:00 UTC; this should be at least 20 minutes later
12-
DB_UPDATE_TIME_UTC=01:20
10+
# Update schedule: anchor time (HH:MM) + interval in hours.
11+
# Interval must divide 24 evenly (1, 2, 3, 4, 6, 8, 12, 24).
12+
# Default = every 6h starting at 00:20 UTC -> runs at 00:20, 06:20, 12:20, 18:20.
13+
# Upstream mergedipdb cron targets 01:00 UTC but GitHub Actions schedule
14+
# latency can push the actual release anywhere from ~01:05 to ~06:00+ UTC,
15+
# so we poll 4x/day to catch it whenever it lands.
16+
DB_UPDATE_TIME_UTC=00:20
17+
DB_UPDATE_INTERVAL_HOURS=6
1318

1419
# Site domain name (used for display/metadata)
1520
SITE_DOMAIN=localhost

deploy/ansible/roles/app/templates/env.j2

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
LISTEN_ADDR=0.0.0.0:3000
22
DB_PATH=/data/Merged-IP.mmdb
33
DB_UPDATE_URL={{ db_update_url }}
4-
DB_UPDATE_TIME_UTC={{ db_update_time_utc | default('01:20') }}
4+
DB_UPDATE_TIME_UTC={{ db_update_time_utc | default('00:20') }}
5+
DB_UPDATE_INTERVAL_HOURS={{ db_update_interval_hours | default('6') }}
6+
RUST_LOG={{ rust_log | default('rustyip=info,warn') }}
57
SITE_DOMAIN={{ site_domain }}
68
{% if ipv4_domain %}
79
IPV4_DOMAIN={{ ipv4_domain }}

docker-compose.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ services:
2121
- LISTEN_ADDR=0.0.0.0:3000
2222
- DB_PATH=/data/Merged-IP.mmdb
2323
- DB_UPDATE_URL=${DB_UPDATE_URL:-https://github.com/NetworkCats/Merged-IP-Data/releases/latest/download/Merged-IP.mmdb}
24-
- DB_UPDATE_TIME_UTC=${DB_UPDATE_TIME_UTC:-01:20}
24+
- DB_UPDATE_TIME_UTC=${DB_UPDATE_TIME_UTC:-00:20}
25+
- DB_UPDATE_INTERVAL_HOURS=${DB_UPDATE_INTERVAL_HOURS:-6}
2526
- SITE_DOMAIN=${SITE_DOMAIN:-localhost}
2627
- IPV4_DOMAIN=${IPV4_DOMAIN:-}
2728
volumes:

src/config.rs

Lines changed: 84 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@ pub struct Config {
77
pub listen_addr: SocketAddr,
88
pub db_path: String,
99
pub db_update_url: String,
10-
/// Daily UTC time to check for database updates, as (hour, minute).
10+
/// Anchor UTC time for the update schedule, as (hour, minute).
11+
/// Updates run at this time and every `db_update_interval_hours` thereafter.
1112
pub db_update_time_utc: (u8, u8),
13+
/// Hours between update ticks. Must divide 24 evenly (1, 2, 3, 4, 6, 8, 12, 24).
14+
pub db_update_interval_hours: u8,
1215
pub site_domain: String,
1316
pub ipv4_domain: String,
1417
pub dev_mode: bool,
@@ -32,6 +35,22 @@ fn parse_time_utc(s: &str) -> (u8, u8) {
3235
(hour, minute)
3336
}
3437

38+
/// Parses the interval-hours string, panicking on invalid input or values that do not divide 24 evenly.
39+
fn parse_interval_hours(s: &str) -> u8 {
40+
let v: u8 = s
41+
.parse()
42+
.expect("DB_UPDATE_INTERVAL_HOURS must be a valid integer 1-24");
43+
assert!(
44+
(1..=24).contains(&v),
45+
"DB_UPDATE_INTERVAL_HOURS must be 1-24"
46+
);
47+
assert!(
48+
24 % v == 0,
49+
"DB_UPDATE_INTERVAL_HOURS must divide 24 evenly (1, 2, 3, 4, 6, 8, 12, 24)"
50+
);
51+
v
52+
}
53+
3554
impl Config {
3655
pub fn from_env() -> Self {
3756
let listen_addr = env::var("LISTEN_ADDR")
@@ -47,7 +66,11 @@ impl Config {
4766
});
4867

4968
let db_update_time_utc =
50-
parse_time_utc(&env::var("DB_UPDATE_TIME_UTC").unwrap_or_else(|_| "01:20".to_string()));
69+
parse_time_utc(&env::var("DB_UPDATE_TIME_UTC").unwrap_or_else(|_| "00:20".to_string()));
70+
71+
let db_update_interval_hours = parse_interval_hours(
72+
&env::var("DB_UPDATE_INTERVAL_HOURS").unwrap_or_else(|_| "6".to_string()),
73+
);
5174

5275
let site_domain = env::var("SITE_DOMAIN").unwrap_or_else(|_| "localhost".to_string());
5376

@@ -62,6 +85,7 @@ impl Config {
6285
db_path,
6386
db_update_url,
6487
db_update_time_utc,
88+
db_update_interval_hours,
6589
site_domain,
6690
ipv4_domain,
6791
dev_mode,
@@ -89,6 +113,7 @@ mod tests {
89113
"DB_PATH",
90114
"DB_UPDATE_URL",
91115
"DB_UPDATE_TIME_UTC",
116+
"DB_UPDATE_INTERVAL_HOURS",
92117
"SITE_DOMAIN",
93118
"IPV4_DOMAIN",
94119
"DEV_MODE",
@@ -120,7 +145,8 @@ mod tests {
120145
assert_eq!(config.listen_addr.to_string(), "0.0.0.0:3000");
121146
assert_eq!(config.db_path, "data/Merged-IP.mmdb");
122147
assert!(config.db_update_url.contains("Merged-IP.mmdb"));
123-
assert_eq!(config.db_update_time_utc, (1, 20));
148+
assert_eq!(config.db_update_time_utc, (0, 20));
149+
assert_eq!(config.db_update_interval_hours, 6);
124150
assert_eq!(config.site_domain, "localhost");
125151
assert!(config.ipv4_domain.is_empty());
126152
assert!(!config.dev_mode);
@@ -190,6 +216,22 @@ mod tests {
190216
unsafe { remove_var("DB_UPDATE_TIME_UTC") };
191217
}
192218

219+
#[test]
220+
fn custom_update_interval() {
221+
let _guard = lock_env();
222+
// SAFETY: ENV_LOCK is held.
223+
unsafe {
224+
clear_config_vars();
225+
set_var("DB_UPDATE_INTERVAL_HOURS", "12");
226+
}
227+
228+
let config = Config::from_env();
229+
assert_eq!(config.db_update_interval_hours, 12);
230+
231+
// SAFETY: ENV_LOCK is held.
232+
unsafe { remove_var("DB_UPDATE_INTERVAL_HOURS") };
233+
}
234+
193235
#[test]
194236
fn custom_site_domain() {
195237
let _guard = lock_env();
@@ -315,4 +357,43 @@ mod tests {
315357

316358
let _config = Config::from_env();
317359
}
360+
361+
#[test]
362+
#[should_panic(expected = "DB_UPDATE_INTERVAL_HOURS must divide 24 evenly")]
363+
fn update_interval_non_divisor_panics() {
364+
let _guard = lock_env();
365+
// SAFETY: ENV_LOCK is held.
366+
unsafe {
367+
clear_config_vars();
368+
set_var("DB_UPDATE_INTERVAL_HOURS", "5");
369+
}
370+
371+
let _config = Config::from_env();
372+
}
373+
374+
#[test]
375+
#[should_panic(expected = "DB_UPDATE_INTERVAL_HOURS must be 1-24")]
376+
fn update_interval_zero_panics() {
377+
let _guard = lock_env();
378+
// SAFETY: ENV_LOCK is held.
379+
unsafe {
380+
clear_config_vars();
381+
set_var("DB_UPDATE_INTERVAL_HOURS", "0");
382+
}
383+
384+
let _config = Config::from_env();
385+
}
386+
387+
#[test]
388+
#[should_panic(expected = "DB_UPDATE_INTERVAL_HOURS must be a valid integer 1-24")]
389+
fn update_interval_invalid_panics() {
390+
let _guard = lock_env();
391+
// SAFETY: ENV_LOCK is held.
392+
unsafe {
393+
clear_config_vars();
394+
set_var("DB_UPDATE_INTERVAL_HOURS", "not-a-number");
395+
}
396+
397+
let _config = Config::from_env();
398+
}
318399
}

src/main.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ async fn main() {
7676
init_path,
7777
init_url,
7878
config.db_update_time_utc,
79+
config.db_update_interval_hours,
7980
init_cancel,
8081
)
8182
.await;

src/updater.rs

Lines changed: 74 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -104,38 +104,49 @@ pub fn spawn_updater(
104104
db_path: PathBuf,
105105
update_url: String,
106106
update_time_utc: (u8, u8),
107+
interval_hours: u8,
107108
cancel: CancellationToken,
108109
) -> JoinHandle<()> {
109110
tokio::spawn(run_updater(
110111
shared_db,
111112
db_path,
112113
update_url,
113114
update_time_utc,
115+
interval_hours,
114116
cancel,
115117
))
116118
}
117119

118120
/// Returns the number of seconds from `now_secs` (seconds since UNIX epoch)
119-
/// until the next occurrence of the given UTC `(hour, minute)`.
120-
fn secs_until_next(now_secs: u64, hour: u8, minute: u8) -> u64 {
121+
/// until the next tick. Ticks occur at `(anchor_hour, anchor_minute)` UTC and
122+
/// every `interval_hours` thereafter. `interval_hours` must divide 24 evenly.
123+
fn secs_until_next_tick(
124+
now_secs: u64,
125+
anchor_hour: u8,
126+
anchor_minute: u8,
127+
interval_hours: u8,
128+
) -> u64 {
121129
const SECS_PER_DAY: u64 = 86_400;
130+
let interval = u64::from(interval_hours) * 3600;
131+
let anchor = u64::from(anchor_hour) * 3600 + u64::from(anchor_minute) * 60;
122132
let time_of_day = now_secs % SECS_PER_DAY;
123-
let target = u64::from(hour) * 3600 + u64::from(minute) * 60;
124-
if target > time_of_day {
125-
target - time_of_day
133+
let offset_from_anchor = if time_of_day >= anchor {
134+
time_of_day - anchor
126135
} else {
127-
// Target time already passed today; schedule for tomorrow.
128-
SECS_PER_DAY - time_of_day + target
129-
}
136+
SECS_PER_DAY - (anchor - time_of_day)
137+
};
138+
let next_boundary = ((offset_from_anchor / interval) + 1) * interval;
139+
next_boundary - offset_from_anchor
130140
}
131141

132-
/// Runs the daily database updater loop. Sleeps until the configured UTC time
133-
/// each day, then downloads and swaps in a fresh database.
142+
/// Runs the database updater loop. Sleeps until the next scheduled tick, then
143+
/// downloads and hot-swaps a fresh database.
134144
pub async fn run_updater(
135145
shared_db: SharedDb,
136146
db_path: PathBuf,
137147
update_url: String,
138148
update_time_utc: (u8, u8),
149+
interval_hours: u8,
139150
cancel: CancellationToken,
140151
) {
141152
let (hour, minute) = update_time_utc;
@@ -144,13 +155,15 @@ pub async fn run_updater(
144155
.duration_since(UNIX_EPOCH)
145156
.expect("system clock before UNIX epoch")
146157
.as_secs();
147-
let delay = Duration::from_secs(secs_until_next(now_secs, hour, minute));
158+
let delay =
159+
Duration::from_secs(secs_until_next_tick(now_secs, hour, minute, interval_hours));
148160
info!(
149-
"next database update in {}h {}m (at {:02}:{:02} UTC)",
161+
"next database update in {}h {}m (anchor {:02}:{:02} UTC, every {}h)",
150162
delay.as_secs() / 3600,
151163
(delay.as_secs() % 3600) / 60,
152164
hour,
153165
minute,
166+
interval_hours,
154167
);
155168
tokio::select! {
156169
() = tokio::time::sleep(delay) => {}
@@ -172,7 +185,12 @@ async fn update_db(
172185
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
173186
let reader = download_db(update_url, db_path).await?;
174187
shared_db.store(Arc::new(Some(reader)));
175-
info!("database updated successfully");
188+
let size = fs::metadata(db_path)
189+
.await
190+
.ok()
191+
.map(|m| m.len())
192+
.unwrap_or(0);
193+
info!("database updated successfully ({size} bytes)");
176194
Ok(())
177195
}
178196

@@ -279,7 +297,8 @@ mod tests {
279297
shared_db,
280298
PathBuf::from("/nonexistent/path.mmdb"),
281299
"https://invalid.example.com/db.mmdb".to_string(),
282-
(1, 20),
300+
(0, 20),
301+
6,
283302
cancel.clone(),
284303
);
285304

@@ -293,31 +312,57 @@ mod tests {
293312
}
294313

295314
#[test]
296-
fn secs_until_next_later_today() {
297-
// 00:00:00 UTC, target 01:20 UTC => 1h 20m = 4800s
298-
assert_eq!(secs_until_next(0, 1, 20), 4800);
315+
fn secs_until_next_tick_daily_from_midnight() {
316+
// 00:00 UTC, anchor 01:20, interval 24h => 1h 20m = 4800s.
317+
assert_eq!(secs_until_next_tick(0, 1, 20, 24), 4800);
318+
}
319+
320+
#[test]
321+
fn secs_until_next_tick_daily_wraps_to_tomorrow() {
322+
// 02:00 UTC (7200s), anchor 01:20 (4800s), interval 24h
323+
// => 86400 - 7200 + 4800 = 84000s.
324+
assert_eq!(secs_until_next_tick(7200, 1, 20, 24), 84000);
325+
}
326+
327+
#[test]
328+
fn secs_until_next_tick_daily_exact_time_wraps() {
329+
// Exactly at the anchor with 24h interval => schedule for next day.
330+
let target_secs = 3600 + 20 * 60;
331+
assert_eq!(secs_until_next_tick(target_secs, 1, 20, 24), 86400);
332+
}
333+
334+
#[test]
335+
fn secs_until_next_tick_every_six_hours_at_midnight() {
336+
// 00:00 UTC with anchor 00:20 and interval 6h => next tick in 20m = 1200s.
337+
assert_eq!(secs_until_next_tick(0, 0, 20, 6), 1200);
338+
}
339+
340+
#[test]
341+
fn secs_until_next_tick_every_six_hours_just_after_tick() {
342+
// 00:21 UTC (1260s), anchor 00:20, interval 6h
343+
// => next tick is 06:20 UTC = 6h - 1m = 21540s.
344+
let now_secs = 20 * 60 + 60; // 00:21
345+
assert_eq!(secs_until_next_tick(now_secs, 0, 20, 6), 21540);
299346
}
300347

301348
#[test]
302-
fn secs_until_next_wraps_to_tomorrow() {
303-
// 02:00:00 UTC (7200s into the day), target 01:20 (4800s into the day)
304-
// => should wrap to next day: 86400 - 7200 + 4800 = 84000s
305-
let now_secs = 7200; // midnight + 2 hours
306-
assert_eq!(secs_until_next(now_secs, 1, 20), 84000);
349+
fn secs_until_next_tick_every_six_hours_at_tick() {
350+
// Exactly at a tick (00:20) with 6h interval => schedule for 6h later.
351+
let target_secs = 20 * 60;
352+
assert_eq!(secs_until_next_tick(target_secs, 0, 20, 6), 6 * 3600);
307353
}
308354

309355
#[test]
310-
fn secs_until_next_exact_time_wraps_to_tomorrow() {
311-
// Exactly at the target time => should schedule for next day (full 24h).
312-
let target_secs = 1 * 3600 + 20 * 60; // 01:20 = 4800s
313-
assert_eq!(secs_until_next(target_secs, 1, 20), 86400);
356+
fn secs_until_next_tick_every_six_hours_before_anchor() {
357+
// 00:10 UTC (600s), anchor 00:20, interval 6h => next tick in 10m = 600s.
358+
assert_eq!(secs_until_next_tick(600, 0, 20, 6), 600);
314359
}
315360

316361
#[test]
317-
fn secs_until_next_midnight_target() {
318-
// 23:00:00 UTC, target 00:00 UTC => 1 hour = 3600s
362+
fn secs_until_next_tick_midnight_target() {
363+
// 23:00 UTC, anchor 00:00, interval 24h => 1h = 3600s.
319364
let now_secs = 23 * 3600;
320-
assert_eq!(secs_until_next(now_secs, 0, 0), 3600);
365+
assert_eq!(secs_until_next_tick(now_secs, 0, 0, 24), 3600);
321366
}
322367

323368
#[tokio::test]

0 commit comments

Comments
 (0)