Skip to content

Commit d0e408c

Browse files
graylikemeclaude
andcommitted
feat: add equipment-seed scraper subcommand with initial stats data
Add equipment-seed subcommand to bulk-update equipment stats from a JSON file. Supports --force mode (overwrite all) and default mode (fill NULLs only via COALESCE). Includes ~100 common weapons in data/equipment_stats.json. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 6042356 commit d0e408c

3 files changed

Lines changed: 280 additions & 0 deletions

File tree

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
use std::path::Path;
2+
3+
use anyhow::Context;
4+
use rust_decimal::Decimal;
5+
use serde::Deserialize;
6+
use sqlx::Row;
7+
use tracing::{info, warn};
8+
9+
#[derive(Debug, Deserialize)]
10+
pub struct EquipmentStats {
11+
pub slug: String,
12+
pub tonnage: Option<f64>,
13+
pub crits: Option<i32>,
14+
pub damage: Option<String>,
15+
pub heat: Option<i32>,
16+
pub range_min: Option<i32>,
17+
pub range_short: Option<i32>,
18+
pub range_medium: Option<i32>,
19+
pub range_long: Option<i32>,
20+
pub bv: Option<i32>,
21+
}
22+
23+
pub async fn run(file: &Path, database_url: &str, pool_size: u32, force: bool) -> anyhow::Result<()> {
24+
let pool = sqlx::postgres::PgPoolOptions::new()
25+
.max_connections(pool_size)
26+
.connect(database_url)
27+
.await
28+
.context("connecting to database")?;
29+
30+
let content = std::fs::read_to_string(file)
31+
.with_context(|| format!("reading {:?}", file))?;
32+
let entries: Vec<EquipmentStats> =
33+
serde_json::from_str(&content).context("parsing equipment stats JSON")?;
34+
35+
info!(count = entries.len(), "loaded equipment stats entries");
36+
37+
let mut updated = 0u32;
38+
let skipped = 0u32;
39+
let mut not_found = 0u32;
40+
let mut unchanged = 0u32;
41+
42+
for entry in &entries {
43+
let exists = sqlx::query("SELECT id FROM equipment WHERE slug = $1")
44+
.bind(&entry.slug)
45+
.fetch_optional(&pool)
46+
.await?;
47+
48+
let Some(row) = exists else {
49+
warn!(slug = %entry.slug, "no matching equipment row");
50+
not_found += 1;
51+
continue;
52+
};
53+
let eq_id: i32 = row.try_get("id")?;
54+
55+
if force {
56+
let result = sqlx::query(
57+
r#"UPDATE equipment SET
58+
tonnage = $2,
59+
crits = $3,
60+
damage = $4,
61+
heat = $5,
62+
range_min = $6,
63+
range_short = $7,
64+
range_medium = $8,
65+
range_long = $9,
66+
bv = $10,
67+
stats_source = 'seed',
68+
stats_updated_at = now()
69+
WHERE id = $1"#,
70+
)
71+
.bind(eq_id)
72+
.bind(entry.tonnage.map(|t| Decimal::try_from(t).unwrap_or_default()))
73+
.bind(entry.crits)
74+
.bind(&entry.damage)
75+
.bind(entry.heat)
76+
.bind(entry.range_min)
77+
.bind(entry.range_short)
78+
.bind(entry.range_medium)
79+
.bind(entry.range_long)
80+
.bind(entry.bv)
81+
.execute(&pool)
82+
.await?;
83+
84+
if result.rows_affected() > 0 {
85+
updated += 1;
86+
} else {
87+
unchanged += 1;
88+
}
89+
} else {
90+
// Only update NULL columns
91+
let result = sqlx::query(
92+
r#"UPDATE equipment SET
93+
tonnage = COALESCE(tonnage, $2),
94+
crits = COALESCE(crits, $3),
95+
damage = COALESCE(damage, $4),
96+
heat = COALESCE(heat, $5),
97+
range_min = COALESCE(range_min, $6),
98+
range_short = COALESCE(range_short, $7),
99+
range_medium = COALESCE(range_medium, $8),
100+
range_long = COALESCE(range_long, $9),
101+
bv = COALESCE(bv, $10),
102+
stats_source = COALESCE(stats_source, 'seed'),
103+
stats_updated_at = COALESCE(stats_updated_at, now())
104+
WHERE id = $1
105+
AND (tonnage IS NULL OR crits IS NULL OR damage IS NULL
106+
OR heat IS NULL OR range_min IS NULL OR range_short IS NULL
107+
OR range_medium IS NULL OR range_long IS NULL OR bv IS NULL)"#,
108+
)
109+
.bind(eq_id)
110+
.bind(entry.tonnage.map(|t| Decimal::try_from(t).unwrap_or_default()))
111+
.bind(entry.crits)
112+
.bind(&entry.damage)
113+
.bind(entry.heat)
114+
.bind(entry.range_min)
115+
.bind(entry.range_short)
116+
.bind(entry.range_medium)
117+
.bind(entry.range_long)
118+
.bind(entry.bv)
119+
.execute(&pool)
120+
.await?;
121+
122+
if result.rows_affected() > 0 {
123+
updated += 1;
124+
} else {
125+
unchanged += 1;
126+
}
127+
}
128+
}
129+
130+
info!(
131+
updated,
132+
skipped,
133+
not_found,
134+
unchanged,
135+
"equipment seed complete"
136+
);
137+
138+
// Suppress unused variable warning
139+
let _ = skipped;
140+
141+
Ok(())
142+
}

crates/scraper/src/main.rs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
mod db;
2+
mod equipment_seed;
23
mod mul;
34
mod parse;
45
mod seed;
@@ -69,6 +70,25 @@ enum Command {
6970
types: Vec<u32>,
7071
},
7172

73+
/// Seed equipment stats from a JSON file into the database.
74+
EquipmentSeed {
75+
/// Path to the equipment stats JSON file.
76+
#[arg(long, value_name = "FILE")]
77+
file: PathBuf,
78+
79+
/// Override DATABASE_URL (defaults to env var).
80+
#[arg(long, env = "DATABASE_URL")]
81+
database_url: String,
82+
83+
/// Maximum DB connections in pool.
84+
#[arg(long, default_value_t = 5)]
85+
pool_size: u32,
86+
87+
/// Overwrite all stats columns, not just NULLs.
88+
#[arg(long)]
89+
force: bool,
90+
},
91+
7292
/// Import previously-fetched MUL data from local files into the database.
7393
MulImport {
7494
/// Directory containing fetched MUL data.
@@ -122,6 +142,14 @@ async fn main() -> anyhow::Result<()> {
122142
} => {
123143
run_megamek(zip, &database_url, &version, pool_size, max_errors).await
124144
}
145+
Command::EquipmentSeed {
146+
file,
147+
database_url,
148+
pool_size,
149+
force,
150+
} => {
151+
equipment_seed::run(&file, &database_url, pool_size, force).await
152+
}
125153
Command::MulFetch {
126154
output_dir,
127155
delay_ms,

0 commit comments

Comments
 (0)