From 8231402338daf5ec1f9693557b138a2abe83ab16 Mon Sep 17 00:00:00 2001 From: Jake Bailey Date: Mon, 11 May 2026 23:58:09 -0700 Subject: [PATCH 1/2] Add mledb to sprocket migration guardrails --- .../mledb-sprocket/00_guard_schema.sql | 236 +++++++++++++++ .../migration/mledb-sprocket/01_preflight.sql | 150 ++++++++++ .../mledb-sprocket/02_capture_baseline.sql | 34 +++ .../03_validate_equivalence.sql | 269 ++++++++++++++++++ .../mledb-sprocket/04_domain_coverage.sql | 88 ++++++ .../90_rollback_bridge_mapped_rows.sql | 99 +++++++ ...ledb-sprocket-migration-validation-plan.md | 100 +++++++ .../migration/run-mledb-sprocket-migration.sh | 187 ++++++++++++ 8 files changed, 1163 insertions(+) create mode 100644 queries/migration/mledb-sprocket/00_guard_schema.sql create mode 100644 queries/migration/mledb-sprocket/01_preflight.sql create mode 100644 queries/migration/mledb-sprocket/02_capture_baseline.sql create mode 100644 queries/migration/mledb-sprocket/03_validate_equivalence.sql create mode 100644 queries/migration/mledb-sprocket/04_domain_coverage.sql create mode 100644 queries/migration/mledb-sprocket/90_rollback_bridge_mapped_rows.sql create mode 100644 reports/mledb-sprocket-migration-validation-plan.md create mode 100755 scripts/migration/run-mledb-sprocket-migration.sh diff --git a/queries/migration/mledb-sprocket/00_guard_schema.sql b/queries/migration/mledb-sprocket/00_guard_schema.sql new file mode 100644 index 000000000..4996f041c --- /dev/null +++ b/queries/migration/mledb-sprocket/00_guard_schema.sql @@ -0,0 +1,236 @@ +\set ON_ERROR_STOP on + +CREATE SCHEMA IF NOT EXISTS migration_guard; + +CREATE TABLE IF NOT EXISTS migration_guard.runs ( + run_id uuid PRIMARY KEY, + description text NOT NULL DEFAULT '', + git_sha text, + started_at timestamptz NOT NULL DEFAULT now(), + completed_at timestamptz, + status text NOT NULL DEFAULT 'baseline-captured', + notes text +); + +CREATE TABLE IF NOT EXISTS migration_guard.sprocket_table_snapshots ( + run_id uuid NOT NULL REFERENCES migration_guard.runs(run_id), + schema_name text NOT NULL, + table_name text NOT NULL, + row_count bigint NOT NULL, + table_hash text NOT NULL, + has_primary_key boolean NOT NULL, + captured_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (run_id, schema_name, table_name) +); + +CREATE TABLE IF NOT EXISTS migration_guard.sprocket_row_snapshots ( + run_id uuid NOT NULL REFERENCES migration_guard.runs(run_id), + schema_name text NOT NULL, + table_name text NOT NULL, + primary_key jsonb NOT NULL, + row_hash text NOT NULL, + captured_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (run_id, schema_name, table_name, primary_key) +); + +CREATE OR REPLACE FUNCTION migration_guard.capture_sprocket_baseline( + p_run_id uuid, + p_description text DEFAULT '', + p_git_sha text DEFAULT NULL +) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + tbl record; + pk_expr text; + pk_cols int; + table_hash text; + row_count bigint; +BEGIN + INSERT INTO migration_guard.runs(run_id, description, git_sha) + VALUES (p_run_id, p_description, p_git_sha) + ON CONFLICT (run_id) DO UPDATE + SET description = EXCLUDED.description, + git_sha = EXCLUDED.git_sha, + started_at = now(), + completed_at = NULL, + status = 'baseline-captured', + notes = NULL; + + DELETE FROM migration_guard.sprocket_row_snapshots WHERE run_id = p_run_id; + DELETE FROM migration_guard.sprocket_table_snapshots WHERE run_id = p_run_id; + + FOR tbl IN + SELECT c.oid AS relation_oid, n.nspname AS schema_name, c.relname AS table_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'sprocket' + AND c.relkind IN ('r', 'p') + ORDER BY n.nspname, c.relname + LOOP + EXECUTE format( + 'SELECT count(*)::bigint, md5(coalesce(string_agg(to_jsonb(t)::text, E''\n'' ORDER BY to_jsonb(t)::text), '''')) FROM %I.%I t', + tbl.schema_name, + tbl.table_name + ) + INTO row_count, table_hash; + + SELECT count(*), + string_agg( + format('%L, t.%I', a.attname, a.attname), + ', ' ORDER BY k.ordinality + ) + INTO pk_cols, pk_expr + FROM pg_index i + JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, ordinality) ON true + JOIN pg_attribute a + ON a.attrelid = i.indrelid + AND a.attnum = k.attnum + WHERE i.indrelid = tbl.relation_oid + AND i.indisprimary; + + INSERT INTO migration_guard.sprocket_table_snapshots( + run_id, + schema_name, + table_name, + row_count, + table_hash, + has_primary_key + ) + VALUES ( + p_run_id, + tbl.schema_name, + tbl.table_name, + row_count, + table_hash, + pk_cols > 0 + ); + + IF pk_cols > 0 THEN + EXECUTE format( + 'INSERT INTO migration_guard.sprocket_row_snapshots(run_id, schema_name, table_name, primary_key, row_hash) + SELECT $1, %L, %L, jsonb_build_object(%s), md5(to_jsonb(t)::text) + FROM %I.%I t', + tbl.schema_name, + tbl.table_name, + pk_expr, + tbl.schema_name, + tbl.table_name + ) + USING p_run_id; + END IF; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION migration_guard.validate_sprocket_baseline(p_run_id uuid) +RETURNS TABLE ( + issue_type text, + schema_name text, + table_name text, + primary_key jsonb, + before_hash text, + after_hash text, + detail text +) +LANGUAGE plpgsql +AS $$ +DECLARE + tbl record; + pk_expr text; +BEGIN + FOR tbl IN + SELECT s.schema_name, + s.table_name, + c.oid AS relation_oid, + s.has_primary_key, + s.row_count, + s.table_hash + FROM migration_guard.sprocket_table_snapshots s + JOIN pg_namespace n ON n.nspname = s.schema_name + JOIN pg_class c ON c.relnamespace = n.oid AND c.relname = s.table_name + WHERE s.run_id = p_run_id + ORDER BY s.schema_name, s.table_name + LOOP + IF tbl.has_primary_key THEN + SELECT string_agg( + format('%L, t.%I', a.attname, a.attname), + ', ' ORDER BY k.ordinality + ) + INTO pk_expr + FROM pg_index i + JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, ordinality) ON true + JOIN pg_attribute a + ON a.attrelid = i.indrelid + AND a.attnum = k.attnum + WHERE i.indrelid = tbl.relation_oid + AND i.indisprimary; + + RETURN QUERY EXECUTE format( + 'WITH current_rows AS ( + SELECT jsonb_build_object(%s) AS primary_key, md5(to_jsonb(t)::text) AS row_hash + FROM %I.%I t + ) + SELECT ''deleted_preexisting_sprocket_row''::text, + s.schema_name, + s.table_name, + s.primary_key, + s.row_hash, + NULL::text, + ''row existed before migration and is now missing''::text + FROM migration_guard.sprocket_row_snapshots s + LEFT JOIN current_rows c ON c.primary_key = s.primary_key + WHERE s.run_id = $1 + AND s.schema_name = %L + AND s.table_name = %L + AND c.primary_key IS NULL + UNION ALL + SELECT ''changed_preexisting_sprocket_row''::text, + s.schema_name, + s.table_name, + s.primary_key, + s.row_hash, + c.row_hash, + ''row existed before migration and its JSON hash changed''::text + FROM migration_guard.sprocket_row_snapshots s + JOIN current_rows c ON c.primary_key = s.primary_key + WHERE s.run_id = $1 + AND s.schema_name = %L + AND s.table_name = %L + AND c.row_hash <> s.row_hash', + pk_expr, + tbl.schema_name, + tbl.table_name, + tbl.schema_name, + tbl.table_name, + tbl.schema_name, + tbl.table_name + ) + USING p_run_id; + ELSE + RETURN QUERY EXECUTE format( + 'WITH current_table AS ( + SELECT count(*)::bigint AS row_count, + md5(coalesce(string_agg(to_jsonb(t)::text, E''\n'' ORDER BY to_jsonb(t)::text), '''')) AS table_hash + FROM %I.%I t + ) + SELECT ''changed_sprocket_table_without_pk''::text, + %L::text, + %L::text, + NULL::jsonb, + $2::text, + c.table_hash, + format(''table has no primary key; baseline count/hash was %%s/%%s and current count/hash is %%s/%%s'', $1, $2, c.row_count, c.table_hash)::text + FROM current_table c + WHERE c.row_count <> $1 OR c.table_hash <> $2', + tbl.schema_name, + tbl.table_name, + tbl.schema_name, + tbl.table_name + ) + USING tbl.row_count, tbl.table_hash; + END IF; + END LOOP; +END; +$$; diff --git a/queries/migration/mledb-sprocket/01_preflight.sql b/queries/migration/mledb-sprocket/01_preflight.sql new file mode 100644 index 000000000..ae3359242 --- /dev/null +++ b/queries/migration/mledb-sprocket/01_preflight.sql @@ -0,0 +1,150 @@ +\set ON_ERROR_STOP on + +CREATE TEMP TABLE migration_preflight_issues ( + severity text NOT NULL, + check_name text NOT NULL, + detail text NOT NULL +); + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'required_schema', format('missing schema %s', schema_name) +FROM (VALUES ('mledb'), ('sprocket'), ('mledb_bridge')) AS required(schema_name) +WHERE NOT EXISTS ( + SELECT 1 + FROM information_schema.schemata s + WHERE s.schema_name = required.schema_name +); + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'required_table', format('missing table %s.%s', schema_name, table_name) +FROM ( + VALUES + ('mledb', 'division'), + ('mledb', 'team'), + ('mledb', 'team_branding'), + ('mledb', 'player'), + ('mledb', 'season'), + ('mledb', 'match'), + ('mledb', 'fixture'), + ('mledb', 'series'), + ('sprocket', 'organization'), + ('sprocket', 'organization_profile'), + ('sprocket', 'game'), + ('sprocket', 'game_mode'), + ('sprocket', 'franchise'), + ('sprocket', 'franchise_profile'), + ('sprocket', 'game_skill_group'), + ('sprocket', 'team'), + ('sprocket', 'user'), + ('sprocket', 'user_profile'), + ('sprocket', 'user_authentication_account'), + ('sprocket', 'member'), + ('sprocket', 'member_profile'), + ('sprocket', 'player'), + ('sprocket', 'schedule_group'), + ('sprocket', 'schedule_fixture'), + ('sprocket', 'match_parent'), + ('sprocket', 'match'), + ('mledb_bridge', 'division_to_franchise_group'), + ('mledb_bridge', 'team_to_franchise'), + ('mledb_bridge', 'league_to_skill_group'), + ('mledb_bridge', 'player_to_user'), + ('mledb_bridge', 'player_to_player'), + ('mledb_bridge', 'season_to_schedule_group'), + ('mledb_bridge', 'match_to_schedule_group'), + ('mledb_bridge', 'fixture_to_fixture'), + ('mledb_bridge', 'series_to_match_parent') +) AS required(schema_name, table_name) +WHERE NOT EXISTS ( + SELECT 1 + FROM information_schema.tables t + WHERE t.table_schema = required.schema_name + AND t.table_name = required.table_name +); + +TABLE migration_preflight_issues +ORDER BY CASE severity WHEN 'blocker' THEN 0 ELSE 1 END, check_name, detail; + +DO $$ +DECLARE + structural_blocker_count int; +BEGIN + SELECT count(*) INTO structural_blocker_count + FROM migration_preflight_issues + WHERE severity = 'blocker' + AND check_name IN ('required_schema', 'required_table'); + + IF structural_blocker_count > 0 THEN + RAISE EXCEPTION 'mledb -> sprocket preflight found % structural blocker(s)', structural_blocker_count; + END IF; +END; +$$; + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'duplicate_discord_id', format('discord_id %s appears %s times in mledb.player', discord_id, count(*)) +FROM mledb.player +WHERE nullif(discord_id, '') IS NOT NULL +GROUP BY discord_id +HAVING count(*) > 1; + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'duplicate_player_name', format('player name %s appears %s times in mledb.player', name, count(*)) +FROM mledb.player +GROUP BY name +HAVING count(*) > 1; + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'fixture_missing_team_bridge_source', format('fixture %s references missing team home=%s away=%s', f.id, f.home_name, f.away_name) +FROM mledb.fixture f +LEFT JOIN mledb.team ht ON ht.name = f.home_name +LEFT JOIN mledb.team at ON at.name = f.away_name +WHERE ht.name IS NULL OR at.name IS NULL; + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'series_unknown_league', format('series %s has unsupported league %s', id, league) +FROM mledb.series +WHERE fixture_id IS NOT NULL + AND league NOT IN ('FOUNDATION', 'ACADEMY', 'CHAMPION', 'MASTER', 'PREMIER'); + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'series_unknown_mode', format('series %s has unsupported mode %s', id, mode) +FROM mledb.series +WHERE fixture_id IS NOT NULL + AND mode NOT IN ('DOUBLES', 'STANDARD'); + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'blocker', 'player_unknown_league', format('player %s (%s) has unsupported league %s', id, name, league) +FROM mledb.player +WHERE league NOT IN ('FOUNDATION', 'ACADEMY', 'CHAMPION', 'MASTER', 'PREMIER', 'UNKNOWN'); + +INSERT INTO migration_preflight_issues(severity, check_name, detail) +SELECT 'warning', 'bridge_table_not_empty', format('%s.%s already contains %s rows', table_schema, table_name, row_count) +FROM ( + SELECT 'mledb_bridge'::text AS table_schema, 'division_to_franchise_group'::text AS table_name, count(*)::bigint AS row_count FROM mledb_bridge.division_to_franchise_group + UNION ALL SELECT 'mledb_bridge', 'team_to_franchise', count(*) FROM mledb_bridge.team_to_franchise + UNION ALL SELECT 'mledb_bridge', 'league_to_skill_group', count(*) FROM mledb_bridge.league_to_skill_group + UNION ALL SELECT 'mledb_bridge', 'player_to_user', count(*) FROM mledb_bridge.player_to_user + UNION ALL SELECT 'mledb_bridge', 'player_to_player', count(*) FROM mledb_bridge.player_to_player + UNION ALL SELECT 'mledb_bridge', 'season_to_schedule_group', count(*) FROM mledb_bridge.season_to_schedule_group + UNION ALL SELECT 'mledb_bridge', 'match_to_schedule_group', count(*) FROM mledb_bridge.match_to_schedule_group + UNION ALL SELECT 'mledb_bridge', 'fixture_to_fixture', count(*) FROM mledb_bridge.fixture_to_fixture + UNION ALL SELECT 'mledb_bridge', 'series_to_match_parent', count(*) FROM mledb_bridge.series_to_match_parent +) bridge_counts +WHERE row_count > 0; + +TABLE migration_preflight_issues +ORDER BY CASE severity WHEN 'blocker' THEN 0 ELSE 1 END, check_name, detail; + +DO $$ +DECLARE + blocker_count int; +BEGIN + SELECT count(*) INTO blocker_count + FROM migration_preflight_issues + WHERE severity = 'blocker'; + + IF blocker_count > 0 THEN + RAISE EXCEPTION 'mledb -> sprocket preflight found % blocker(s)', blocker_count; + END IF; +END; +$$; diff --git a/queries/migration/mledb-sprocket/02_capture_baseline.sql b/queries/migration/mledb-sprocket/02_capture_baseline.sql new file mode 100644 index 000000000..574d71827 --- /dev/null +++ b/queries/migration/mledb-sprocket/02_capture_baseline.sql @@ -0,0 +1,34 @@ +\set ON_ERROR_STOP on + +\if :{?migration_run_id} +\else + \echo 'missing required psql variable: migration_run_id' + \quit 2 +\endif + +\if :{?migration_description} +\else + \set migration_description 'mledb to sprocket migration baseline' +\endif + +\if :{?migration_git_sha} +\else + \set migration_git_sha '' +\endif + +\ir 00_guard_schema.sql + +SELECT migration_guard.capture_sprocket_baseline( + :'migration_run_id'::uuid, + :'migration_description', + nullif(:'migration_git_sha', '') +); + +SELECT run_id, description, git_sha, started_at, status +FROM migration_guard.runs +WHERE run_id = :'migration_run_id'::uuid; + +SELECT schema_name, table_name, row_count, has_primary_key +FROM migration_guard.sprocket_table_snapshots +WHERE run_id = :'migration_run_id'::uuid +ORDER BY schema_name, table_name; diff --git a/queries/migration/mledb-sprocket/03_validate_equivalence.sql b/queries/migration/mledb-sprocket/03_validate_equivalence.sql new file mode 100644 index 000000000..1b9fda45e --- /dev/null +++ b/queries/migration/mledb-sprocket/03_validate_equivalence.sql @@ -0,0 +1,269 @@ +\set ON_ERROR_STOP on + +\if :{?migration_run_id} +\else + \echo 'missing required psql variable: migration_run_id' + \quit 2 +\endif + +\ir 00_guard_schema.sql + +CREATE TEMP TABLE migration_validation_issues ( + issue_type text NOT NULL, + source_key text, + source_payload jsonb, + target_payload jsonb, + detail text NOT NULL +); + +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT issue_type, + concat(schema_name, '.', table_name, ':', coalesce(primary_key::text, '')), + jsonb_build_object('before_hash', before_hash), + jsonb_build_object('after_hash', after_hash), + detail +FROM migration_guard.validate_sprocket_baseline(:'migration_run_id'::uuid); + +WITH source_rows AS ( + SELECT d.name, d.conference + FROM mledb.division d + WHERE d.conference <> 'META' +), +target_rows AS ( + SELECT b.divison AS name, parent_profile.name AS conference + FROM mledb_bridge.division_to_franchise_group b + JOIN sprocket.franchise_group fg ON fg.id = b."franchiseGroupId" + JOIN sprocket.franchise_group parent_fg ON parent_fg.id = fg."parentGroupId" + JOIN sprocket.franchise_group_profile parent_profile ON parent_profile."groupId" = parent_fg.id +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'division_mismatch', + coalesce(s.name, t.name), + to_jsonb(s), + to_jsonb(t), + 'mledb.division must map to one sprocket franchise group with the same name and conference' +FROM source_rows s +FULL JOIN target_rows t USING (name) +WHERE s IS NULL + OR t IS NULL + OR (CASE s.conference WHEN 'ORANGE' THEN 'Orange' WHEN 'BLUE' THEN 'Blue' ELSE s.conference END) IS DISTINCT FROM t.conference; + +WITH source_rows AS ( + SELECT t.name, t.callsign, tb.primary_color, tb.secondary_color + FROM mledb.team t + LEFT JOIN mledb.team_branding tb ON tb.team_name = t.name + WHERE t.division_name <> 'Meta' +), +target_rows AS ( + SELECT b.team AS name, fp.code AS callsign, fp."primaryColor" AS primary_color, fp."secondaryColor" AS secondary_color + FROM mledb_bridge.team_to_franchise b + JOIN sprocket.franchise_profile fp ON fp."franchiseId" = b."franchiseId" +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'franchise_mismatch', + coalesce(s.name, t.name), + to_jsonb(s), + to_jsonb(t), + 'mledb.team/team_branding must map to one sprocket franchise profile' +FROM source_rows s +FULL JOIN target_rows t USING (name) +WHERE s IS NULL + OR t IS NULL + OR s.callsign IS DISTINCT FROM t.callsign + OR s.primary_color IS DISTINCT FROM t.primary_color + OR s.secondary_color IS DISTINCT FROM t.secondary_color; + +WITH expected AS ( + SELECT p.id AS player_id, + p.name, + nullif(p.discord_id, '') AS discord_id + FROM mledb.player p +), +actual AS ( + SELECT ptu."playerId" AS player_id, + up."displayName" AS name, + uaa."accountId" AS discord_id + FROM mledb_bridge.player_to_user ptu + JOIN sprocket."user" u ON u.id = ptu."userId" + JOIN sprocket.user_profile up ON up."userId" = u.id + JOIN sprocket.member m ON m."userId" = u.id + LEFT JOIN sprocket.user_authentication_account uaa + ON uaa."userId" = u.id + AND uaa."accountType"::text = 'DISCORD' +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'user_identity_mismatch', + coalesce(s.player_id, a.player_id)::text, + to_jsonb(s), + to_jsonb(a), + 'mledb.player identity and Discord account must match sprocket user/member rows' +FROM expected s +FULL JOIN actual a USING (player_id) +WHERE s IS NULL + OR a IS NULL + OR s.name IS DISTINCT FROM a.name + OR s.discord_id IS DISTINCT FROM a.discord_id; + +WITH expected AS ( + SELECT p.id AS player_id, + p.league, + p.salary + FROM mledb.player p + WHERE p.league IN ('FOUNDATION', 'ACADEMY', 'CHAMPION', 'MASTER', 'PREMIER') +), +actual AS ( + SELECT ptu."playerId" AS player_id, + gsgp.code AS league_code, + sp.salary + FROM mledb_bridge.player_to_user ptu + JOIN sprocket.member m ON m."userId" = ptu."userId" + JOIN sprocket.player sp ON sp."memberId" = m.id + JOIN sprocket.game_skill_group gsg ON gsg.id = sp."skillGroupId" + JOIN sprocket.game_skill_group_profile gsgp ON gsgp."skillGroupId" = gsg.id +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'player_roster_mismatch', + coalesce(s.player_id, a.player_id)::text, + to_jsonb(s), + to_jsonb(a), + 'supported MLEDB player league and salary must match Sprocket player rows' +FROM expected s +FULL JOIN actual a USING (player_id) +WHERE s IS NULL + OR a IS NULL + OR (CASE s.league + WHEN 'PREMIER' THEN 'PL' + WHEN 'MASTER' THEN 'ML' + WHEN 'CHAMPION' THEN 'CL' + WHEN 'ACADEMY' THEN 'AL' + WHEN 'FOUNDATION' THEN 'FL' + ELSE s.league + END) IS DISTINCT FROM a.league_code + OR s.salary IS DISTINCT FROM a.salary; + +WITH source_rows AS ( + SELECT season_number, start_date, end_date, concat('Season ', season_number) AS description + FROM mledb.season +), +target_rows AS ( + SELECT b."seasonNumber" AS season_number, sg.start AS start_date, sg."end" AS end_date, sg.description + FROM mledb_bridge.season_to_schedule_group b + JOIN sprocket.schedule_group sg ON sg.id = b."scheduleGroupId" +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'season_schedule_group_mismatch', + coalesce(s.season_number, t.season_number)::text, + to_jsonb(s), + to_jsonb(t), + 'mledb.season must map to one sprocket season schedule group' +FROM source_rows s +FULL JOIN target_rows t USING (season_number) +WHERE s IS NULL + OR t IS NULL + OR s.start_date IS DISTINCT FROM t.start_date + OR s.end_date IS DISTINCT FROM t.end_date + OR s.description IS DISTINCT FROM t.description; + +WITH source_rows AS ( + SELECT f.id AS fixture_id, f.match_id, f.home_name, f.away_name + FROM mledb.fixture f +), +target_rows AS ( + SELECT b."mleFixtureId" AS fixture_id, + mtw."matchId" AS match_id, + home_bridge.team AS home_name, + away_bridge.team AS away_name + FROM mledb_bridge.fixture_to_fixture b + JOIN sprocket.schedule_fixture sf ON sf.id = b."sprocketFixtureId" + JOIN mledb_bridge.match_to_schedule_group mtw ON mtw."weekScheduleGroupId" = sf."scheduleGroupId" + JOIN mledb_bridge.team_to_franchise home_bridge ON home_bridge."franchiseId" = sf."homeFranchiseId" + JOIN mledb_bridge.team_to_franchise away_bridge ON away_bridge."franchiseId" = sf."awayFranchiseId" +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'fixture_mismatch', + coalesce(s.fixture_id, t.fixture_id)::text, + to_jsonb(s), + to_jsonb(t), + 'mledb.fixture must map to one sprocket schedule fixture with same week and teams' +FROM source_rows s +FULL JOIN target_rows t USING (fixture_id) +WHERE s IS NULL + OR t IS NULL + OR s.match_id IS DISTINCT FROM t.match_id + OR s.home_name IS DISTINCT FROM t.home_name + OR s.away_name IS DISTINCT FROM t.away_name; + +WITH source_rows AS ( + SELECT id AS series_id, fixture_id, league, mode + FROM mledb.series + WHERE fixture_id IS NOT NULL + AND league IN ('FOUNDATION', 'ACADEMY', 'CHAMPION', 'MASTER', 'PREMIER') +), +target_rows AS ( + SELECT b."seriesId" AS series_id, + ftf."mleFixtureId" AS fixture_id, + lsg.league, + replace(gm.code, 'RL_', '') AS mode + FROM mledb_bridge.series_to_match_parent b + JOIN mledb_bridge.fixture_to_fixture ftf ON ftf."sprocketFixtureId" = ( + SELECT mp."fixtureId" + FROM sprocket.match_parent mp + WHERE mp.id = b."matchParentId" + ) + JOIN sprocket.match sm ON sm."matchParentId" = b."matchParentId" + JOIN mledb_bridge.league_to_skill_group lsg ON lsg."skillGroupId" = sm."skillGroupId" + JOIN sprocket.game_mode gm ON gm.id = sm."gameModeId" +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'series_match_mismatch', + coalesce(s.series_id, t.series_id)::text, + to_jsonb(s), + to_jsonb(t), + 'mledb.series must map to one sprocket match parent/match with same fixture, league, and mode' +FROM source_rows s +FULL JOIN target_rows t USING (series_id) +WHERE s IS NULL + OR t IS NULL + OR s.fixture_id IS DISTINCT FROM t.fixture_id + OR s.league IS DISTINCT FROM t.league + OR s.mode IS DISTINCT FROM t.mode; + +WITH bridge_counts AS ( + SELECT 'division_to_franchise_group' AS bridge, count(*) AS rows, count(DISTINCT divison) AS source_keys, count(DISTINCT "franchiseGroupId") AS target_keys FROM mledb_bridge.division_to_franchise_group + UNION ALL SELECT 'team_to_franchise', count(*), count(DISTINCT team), count(DISTINCT "franchiseId") FROM mledb_bridge.team_to_franchise + UNION ALL SELECT 'player_to_user', count(*), count(DISTINCT "playerId"), count(DISTINCT "userId") FROM mledb_bridge.player_to_user + UNION ALL SELECT 'player_to_player', count(*), count(DISTINCT "mledPlayerId"), count(DISTINCT "sprocketPlayerId") FROM mledb_bridge.player_to_player + UNION ALL SELECT 'season_to_schedule_group', count(*), count(DISTINCT "seasonNumber"), count(DISTINCT "scheduleGroupId") FROM mledb_bridge.season_to_schedule_group + UNION ALL SELECT 'match_to_schedule_group', count(*), count(DISTINCT "matchId"), count(DISTINCT "weekScheduleGroupId") FROM mledb_bridge.match_to_schedule_group + UNION ALL SELECT 'fixture_to_fixture', count(*), count(DISTINCT "mleFixtureId"), count(DISTINCT "sprocketFixtureId") FROM mledb_bridge.fixture_to_fixture + UNION ALL SELECT 'series_to_match_parent', count(*), count(DISTINCT "seriesId"), count(DISTINCT "matchParentId") FROM mledb_bridge.series_to_match_parent +) +INSERT INTO migration_validation_issues(issue_type, source_key, source_payload, target_payload, detail) +SELECT 'bridge_not_one_to_one', + bridge, + to_jsonb(bridge_counts), + NULL, + 'bridge tables must be one-to-one so semantic comparisons are trustworthy' +FROM bridge_counts +WHERE rows <> source_keys OR rows <> target_keys; + +TABLE migration_validation_issues +ORDER BY issue_type, source_key NULLS FIRST; + +DO $$ +DECLARE + issue_count int; +BEGIN + SELECT count(*) INTO issue_count + FROM migration_validation_issues; + + IF issue_count > 0 THEN + RAISE EXCEPTION 'mledb -> sprocket validation found % issue(s)', issue_count; + END IF; +END; +$$; + +UPDATE migration_guard.runs +SET completed_at = now(), + status = 'validated' +WHERE run_id = :'migration_run_id'::uuid; diff --git a/queries/migration/mledb-sprocket/04_domain_coverage.sql b/queries/migration/mledb-sprocket/04_domain_coverage.sql new file mode 100644 index 000000000..beff75018 --- /dev/null +++ b/queries/migration/mledb-sprocket/04_domain_coverage.sql @@ -0,0 +1,88 @@ +\set ON_ERROR_STOP on + +CREATE TEMP TABLE migration_domain_coverage ( + table_name text PRIMARY KEY, + status text NOT NULL, + validation_gate text NOT NULL, + notes text NOT NULL +); + +INSERT INTO migration_domain_coverage(table_name, status, validation_gate, notes) +VALUES + ('division', 'validated', '03_validate_equivalence.sql division_mismatch', 'League structure: division name and conference.'), + ('team', 'validated', '03_validate_equivalence.sql franchise_mismatch', 'Franchise identity: title and callsign.'), + ('team_branding', 'validated', '03_validate_equivalence.sql franchise_mismatch', 'Franchise colors; logo photo rollback remains bridge/manual-review scoped.'), + ('player', 'validated', '03_validate_equivalence.sql user_identity_mismatch and player_roster_mismatch', 'User/member identity for all players; Sprocket player rows for supported league players.'), + ('season', 'validated', '03_validate_equivalence.sql season_schedule_group_mismatch', 'Season schedule group dates and description.'), + ('match', 'validated', '03_validate_equivalence.sql fixture_mismatch', 'Week schedule group is verified through fixture parentage.'), + ('fixture', 'validated', '03_validate_equivalence.sql fixture_mismatch', 'Fixture week, home franchise, and away franchise.'), + ('series', 'validated', '03_validate_equivalence.sql series_match_mismatch', 'League-play series with fixture_id maps to Sprocket match parent/match.'), + ('player_to_org', 'validated-separately', 'scripts/sql/backfill-user-org-team-permission-from-mledb.sql and reports/issue-726-org-team-permissions.md', 'Org-team permissions have a dedicated backfill and stale legacy grant audit.'), + ('channel_map', 'deferred', 'none yet', 'Discord channel routing needs an explicit target Sprocket configuration/webhook mapping.'), + ('config', 'deferred', 'none yet', 'Legacy configuration needs product-level key mapping before migration.'), + ('draft_order', 'deferred', 'none yet', 'Draft workflow data is not covered by the current Sprocket league-core migration.'), + ('eligibility_data', 'deferred', 'none yet', 'Eligibility history needs a Sprocket target contract.'), + ('elo_data', 'deferred', 'queries/setup/elo-migration-view.sql only', 'Elo has adjacent setup SQL but is not part of this backfill validation.'), + ('footers', 'deferred', 'none yet', 'Presentation/footer content needs an explicit target mapping.'), + ('league_branding', 'deferred', 'none yet', 'League branding must map to skill-group/organization presentation fields before validation.'), + ('player_account', 'deferred', 'none yet', 'Platform account migration must map to member_platform_account.'), + ('player_history', 'deferred', 'none yet', 'Historical roster/player state needs a target history model or archival policy.'), + ('player_stats', 'deferred', 'none yet', 'Stats migration needs row-level stat-line mapping and aggregate reconciliation.'), + ('player_stats_core', 'deferred', 'none yet', 'Core stats migration needs row-level stat-line mapping and aggregate reconciliation.'), + ('psyonix_api_result', 'deferred', 'none yet', 'Raw API cache/result retention needs archival policy.'), + ('salary_cap', 'deferred', 'none yet', 'Salary-cap history needs target mapping beyond current skill-group constants.'), + ('scrim', 'deferred', 'none yet', 'Scrim migration must map to saved_scrim/match_parent and replay submission state.'), + ('series_replay', 'deferred', 'none yet', 'Replay evidence needs target object/storage/submission mapping.'), + ('stream_event', 'deferred', 'none yet', 'Stream events need target scheduled-event/webhook mapping.'), + ('team_core_stats', 'deferred', 'none yet', 'Team stats migration needs stat-line mapping and aggregate reconciliation.'), + ('team_role_usage', 'deferred', 'none yet', 'Role usage needs roster_role_usages mapping and validation.'), + ('team_to_captain', 'deferred', 'none yet', 'Captain/team role semantics need target roster/staff mapping.'); + +CREATE TEMP TABLE migration_domain_coverage_issues AS +SELECT 'unknown_mledb_table' AS issue_type, + t.table_name, + NULL::text AS status, + 'mledb table exists without a migration coverage classification' AS detail +FROM information_schema.tables t +LEFT JOIN migration_domain_coverage c ON c.table_name = t.table_name +WHERE t.table_schema = 'mledb' + AND t.table_type = 'BASE TABLE' + AND c.table_name IS NULL +UNION ALL +SELECT 'classified_missing_table' AS issue_type, + c.table_name, + c.status, + 'coverage manifest references a table not present in mledb' AS detail +FROM migration_domain_coverage c +LEFT JOIN information_schema.tables t + ON t.table_schema = 'mledb' + AND t.table_name = c.table_name + AND t.table_type = 'BASE TABLE' +WHERE t.table_name IS NULL; + +SELECT c.table_name, c.status, c.validation_gate, c.notes +FROM migration_domain_coverage c +ORDER BY + CASE c.status + WHEN 'validated' THEN 0 + WHEN 'validated-separately' THEN 1 + WHEN 'deferred' THEN 2 + ELSE 3 + END, + c.table_name; + +TABLE migration_domain_coverage_issues +ORDER BY issue_type, table_name; + +DO $$ +DECLARE + issue_count int; +BEGIN + SELECT count(*) INTO issue_count + FROM migration_domain_coverage_issues; + + IF issue_count > 0 THEN + RAISE EXCEPTION 'mledb domain coverage manifest has % issue(s)', issue_count; + END IF; +END; +$$; diff --git a/queries/migration/mledb-sprocket/90_rollback_bridge_mapped_rows.sql b/queries/migration/mledb-sprocket/90_rollback_bridge_mapped_rows.sql new file mode 100644 index 000000000..fb96117f4 --- /dev/null +++ b/queries/migration/mledb-sprocket/90_rollback_bridge_mapped_rows.sql @@ -0,0 +1,99 @@ +\set ON_ERROR_STOP on + +\if :{?confirm_rollback} +\else + \echo 'missing required psql variable: confirm_rollback' + \quit 2 +\endif + +SELECT CASE + WHEN :'confirm_rollback' = 'DELETE_MIGRATED_ROWS' THEN 1 + ELSE 1 / 0 +END; + +BEGIN; + +DELETE FROM sprocket.match m +USING mledb_bridge.series_to_match_parent b +WHERE m."matchParentId" = b."matchParentId"; + +DELETE FROM sprocket.match_parent mp +USING mledb_bridge.series_to_match_parent b +WHERE mp.id = b."matchParentId"; + +DELETE FROM sprocket.schedule_fixture sf +USING mledb_bridge.fixture_to_fixture b +WHERE sf.id = b."sprocketFixtureId"; + +DELETE FROM sprocket.schedule_group sg +USING mledb_bridge.match_to_schedule_group b +WHERE sg.id = b."weekScheduleGroupId"; + +DELETE FROM sprocket.schedule_group sg +USING mledb_bridge.season_to_schedule_group b +WHERE sg.id = b."scheduleGroupId"; + +DELETE FROM sprocket.player p +USING mledb_bridge.player_to_player b +WHERE p.id = b."sprocketPlayerId"; + +DELETE FROM sprocket.member_profile mp +USING sprocket.member m, mledb_bridge.player_to_user b +WHERE mp."memberId" = m.id + AND m."userId" = b."userId"; + +DELETE FROM sprocket.member m +USING mledb_bridge.player_to_user b +WHERE m."userId" = b."userId"; + +DELETE FROM sprocket.user_authentication_account uaa +USING mledb_bridge.player_to_user b +WHERE uaa."userId" = b."userId"; + +DELETE FROM sprocket.user_profile up +USING mledb_bridge.player_to_user b +WHERE up."userId" = b."userId"; + +DELETE FROM sprocket."user" u +USING mledb_bridge.player_to_user b +WHERE u.id = b."userId"; + +DELETE FROM sprocket.team t +USING mledb_bridge.team_to_franchise b +WHERE t."franchiseId" = b."franchiseId"; + +DELETE FROM sprocket.franchise_profile fp +USING mledb_bridge.team_to_franchise b +WHERE fp."franchiseId" = b."franchiseId"; + +DELETE FROM sprocket.franchise f +USING mledb_bridge.team_to_franchise b +WHERE f.id = b."franchiseId"; + +DELETE FROM sprocket.franchise_group_profile fgp +USING mledb_bridge.division_to_franchise_group b +WHERE fgp."groupId" = b."franchiseGroupId"; + +DELETE FROM sprocket.franchise_group fg +USING mledb_bridge.division_to_franchise_group b +WHERE fg.id = b."franchiseGroupId"; + +DELETE FROM sprocket.game_skill_group_profile gsgp +USING mledb_bridge.league_to_skill_group b +WHERE gsgp."skillGroupId" = b."skillGroupId"; + +DELETE FROM sprocket.game_skill_group gsg +USING mledb_bridge.league_to_skill_group b +WHERE gsg.id = b."skillGroupId"; + +DELETE FROM mledb_bridge.series_to_match_parent; +DELETE FROM mledb_bridge.fixture_to_fixture; +DELETE FROM mledb_bridge.match_to_schedule_group; +DELETE FROM mledb_bridge.season_to_schedule_group; +DELETE FROM mledb_bridge.player_to_player; +DELETE FROM mledb_bridge.player_to_user; +DELETE FROM mledb_bridge.league_to_skill_group; +DELETE FROM mledb_bridge.team_to_franchise; +DELETE FROM mledb_bridge.division_to_franchise_group; + +COMMIT; diff --git a/reports/mledb-sprocket-migration-validation-plan.md b/reports/mledb-sprocket-migration-validation-plan.md new file mode 100644 index 000000000..2ad91ad63 --- /dev/null +++ b/reports/mledb-sprocket-migration-validation-plan.md @@ -0,0 +1,100 @@ +# MLEDB to Sprocket Migration Validation Plan + +This plan is the operating contract for beginning the MLEDB schema to Sprocket schema migration. The current legacy backfill SQL is still `queries/migration/mledb-migration.sql`; the new guardrail tooling makes each run auditable before it is trusted. + +## Goals + +1. Prove every migrated MLEDB domain row has a semantically equivalent Sprocket row. +2. Prove every Sprocket row that existed before the run still exists and has not changed. +3. Keep a run-scoped artifact trail that can be reviewed after failed or successful runs. +4. Provide a rollback path for rows inserted through `mledb_bridge`, while treating any mutation of pre-existing Sprocket rows as a restore-from-backup event. + +## Runbook + +Use a restored clone of production first. Do not run the legacy backfill on production until the same source dump has a clean validation run on a disposable database. + +```bash +export DATABASE_URL='postgres://...' +scripts/migration/run-mledb-sprocket-migration.sh --step preflight --database-url "$DATABASE_URL" +scripts/migration/run-mledb-sprocket-migration.sh --step coverage --database-url "$DATABASE_URL" + +RUN_ID="$(uuidgen | tr '[:upper:]' '[:lower:]')" +scripts/migration/run-mledb-sprocket-migration.sh --step baseline --run-id "$RUN_ID" --database-url "$DATABASE_URL" + +scripts/migration/run-mledb-sprocket-migration.sh \ + --step legacy-backfill \ + --run-id "$RUN_ID" \ + --database-url "$DATABASE_URL" \ + --allow-legacy-backfill + +scripts/migration/run-mledb-sprocket-migration.sh --step validate --run-id "$RUN_ID" --database-url "$DATABASE_URL" +``` + +The runner writes logs under `artifacts/mledb-sprocket-migration//`. + +## Validation Gates + +Preflight must have zero blockers: + +- Required `mledb`, `mledb_bridge`, and `sprocket` schemas and tables exist. +- MLEDB player identity inputs are unique where Sprocket uniqueness requires it. +- Fixtures reference known teams. +- Migrated series use supported leagues and game modes. +- Bridge tables are reported if they are not empty before a run. + +Domain coverage must have zero manifest issues: + +- Every live `mledb` base table must appear in `04_domain_coverage.sql`. +- Every manifest entry must point at a live `mledb` base table. +- `validated` means covered by `03_validate_equivalence.sql`. +- `validated-separately` means covered by a separate backfill/audit path. +- `deferred` means intentionally not covered by the current league-core migration and cannot be claimed as migrated until a target mapping and validation gate are added. + +Baseline preservation must have zero issues: + +- `migration_guard.capture_sprocket_baseline` stores every existing Sprocket table count/hash. +- For tables with primary keys, it stores every pre-existing row key and JSON row hash. +- `migration_guard.validate_sprocket_baseline` fails if any pre-existing row is deleted or changed. +- Tables without primary keys are protected by whole-table count/hash comparison. + +Semantic equivalence must have zero issues: + +- `mledb.division` to Sprocket franchise groups by name and conference. +- `mledb.team` and `mledb.team_branding` to Sprocket franchise profiles by title/code/colors. +- `mledb.player` to Sprocket user, Discord auth account, and member rows by name and account id. +- Supported-league `mledb.player` rows to Sprocket player rows by league and salary. +- `mledb.season` to Sprocket season schedule groups by dates and description. +- `mledb.fixture` to Sprocket schedule fixtures by week and home/away franchises. +- `mledb.series` to Sprocket match parent/match rows by fixture, league, and game mode. +- Every bridge table used for validation must be one-to-one on source and target keys. + +## Error Handling + +Stop immediately on any failed step. Do not proceed from a failed preflight into baseline or backfill; do not proceed from a failed validation into cutover. + +Classify failures this way: + +- Preflight blocker: fix source data, schema state, or the migration SQL before backfill. +- Baseline preservation issue: assume existing Sprocket data was changed or deleted. Use a database restore or point-in-time recovery; do not use bridge-row rollback as the only recovery. +- Semantic mismatch: inspect the row payloads emitted by `03_validate_equivalence.sql`, patch the backfill mapping, restore the disposable DB, and rerun from preflight. +- Non-empty bridge warning: either use a fresh restored database or intentionally validate the prior bridge state before reusing it. + +## Rollback + +The preferred rollback for a failed rehearsal is to drop and recreate the disposable database from the source dump. + +For a production-like environment where only migrated rows need to be removed, use: + +```bash +scripts/migration/run-mledb-sprocket-migration.sh \ + --step rollback \ + --run-id "$RUN_ID" \ + --database-url "$DATABASE_URL" \ + --confirm-rollback DELETE_MIGRATED_ROWS +``` + +Rollback deletes only rows reachable through `mledb_bridge`, in dependency order, and then clears bridge rows. It is not a substitute for restore/PITR if validation reports changed or deleted pre-existing Sprocket rows. + +## Promotion Criteria + +A migration run can be promoted only when the artifact directory contains passing logs for `preflight`, `baseline`, `legacy-backfill`, and `validate` for the exact dump or database snapshot being promoted. The validation log must show zero rows in `migration_validation_issues`, and the run id must be recorded in the deployment notes. diff --git a/scripts/migration/run-mledb-sprocket-migration.sh b/scripts/migration/run-mledb-sprocket-migration.sh new file mode 100755 index 000000000..600cf417a --- /dev/null +++ b/scripts/migration/run-mledb-sprocket-migration.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SQL_DIR="$ROOT_DIR/queries/migration/mledb-sprocket" +LEGACY_BACKFILL_SQL="$ROOT_DIR/queries/migration/mledb-migration.sql" +ARTIFACT_DIR="$ROOT_DIR/artifacts/mledb-sprocket-migration" + +usage() { + cat <<'USAGE' +Usage: + scripts/migration/run-mledb-sprocket-migration.sh --database-url URL --step STEP [options] + +Steps: + preflight Run read-only readiness checks. + baseline Capture hashes for all existing sprocket rows. + coverage Show the migration coverage classification for every known mledb table. + legacy-backfill Execute queries/migration/mledb-migration.sql. Requires --allow-legacy-backfill. + validate Validate Sprocket row preservation and MLEDB semantic equivalence. + rollback Delete rows reachable through mledb_bridge. Requires --confirm-rollback DELETE_MIGRATED_ROWS. + all Run preflight, coverage, baseline, legacy-backfill, validate. + +Options: + --database-url URL Postgres connection string. Can also use DATABASE_URL. + --run-id UUID Migration guard run id. Defaults to uuidgen output. + --description TEXT Stored with the baseline run. + --artifact-dir PATH Directory for command logs. + --allow-legacy-backfill Required for legacy-backfill/all. + --confirm-rollback TEXT Must be DELETE_MIGRATED_ROWS for rollback. + --help Show this help. +USAGE +} + +database_url="${DATABASE_URL:-}" +step="" +run_id="${MIGRATION_RUN_ID:-}" +description="mledb to sprocket migration" +allow_legacy_backfill=0 +confirm_rollback="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --database-url) + database_url="${2:?missing value for --database-url}" + shift 2 + ;; + --step) + step="${2:?missing value for --step}" + shift 2 + ;; + --run-id) + run_id="${2:?missing value for --run-id}" + shift 2 + ;; + --description) + description="${2:?missing value for --description}" + shift 2 + ;; + --artifact-dir) + ARTIFACT_DIR="${2:?missing value for --artifact-dir}" + shift 2 + ;; + --allow-legacy-backfill) + allow_legacy_backfill=1 + shift + ;; + --confirm-rollback) + confirm_rollback="${2:?missing value for --confirm-rollback}" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$database_url" ]]; then + echo "missing --database-url or DATABASE_URL" >&2 + exit 2 +fi + +if [[ -z "$step" ]]; then + echo "missing --step" >&2 + usage >&2 + exit 2 +fi + +if [[ -z "$run_id" ]]; then + run_id="$(uuidgen | tr '[:upper:]' '[:lower:]')" +fi + +git_sha="$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || true)" +mkdir -p "$ARTIFACT_DIR/$run_id" + +run_psql() { + local name="$1" + local file="$2" + shift 2 + + local log_file="$ARTIFACT_DIR/$run_id/${name}.log" + echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] running $name -> $log_file" + psql "$database_url" \ + --set ON_ERROR_STOP=1 \ + --set "migration_run_id=$run_id" \ + --set "migration_description=$description" \ + --set "migration_git_sha=$git_sha" \ + "$@" \ + --file "$file" \ + 2>&1 | tee "$log_file" +} + +run_preflight() { + run_psql preflight "$SQL_DIR/01_preflight.sql" +} + +run_baseline() { + run_psql baseline "$SQL_DIR/02_capture_baseline.sql" +} + +run_coverage() { + run_psql coverage "$SQL_DIR/04_domain_coverage.sql" +} + +run_legacy_backfill() { + if [[ "$allow_legacy_backfill" -ne 1 ]]; then + echo "legacy-backfill requires --allow-legacy-backfill because the current SQL is not idempotent" >&2 + exit 2 + fi + + run_psql legacy-backfill "$LEGACY_BACKFILL_SQL" +} + +run_validate() { + run_psql validate "$SQL_DIR/03_validate_equivalence.sql" +} + +run_rollback() { + if [[ "$confirm_rollback" != "DELETE_MIGRATED_ROWS" ]]; then + echo "rollback requires --confirm-rollback DELETE_MIGRATED_ROWS" >&2 + exit 2 + fi + + run_psql rollback "$SQL_DIR/90_rollback_bridge_mapped_rows.sql" \ + --set "confirm_rollback=$confirm_rollback" +} + +case "$step" in + preflight) + run_preflight + ;; + baseline) + run_baseline + ;; + coverage) + run_coverage + ;; + legacy-backfill) + run_legacy_backfill + ;; + validate) + run_validate + ;; + rollback) + run_rollback + ;; + all) + run_preflight + run_coverage + run_baseline + run_legacy_backfill + run_validate + ;; + *) + echo "unknown step: $step" >&2 + usage >&2 + exit 2 + ;; +esac + +echo "migration_run_id=$run_id" +echo "artifacts=$ARTIFACT_DIR/$run_id" From 1094d60f9731529634c243edcfc904762e30a33f Mon Sep 17 00:00:00 2001 From: Jake Bailey Date: Tue, 12 May 2026 00:07:31 -0700 Subject: [PATCH 2/2] Assess mledb migration readiness --- .../mledb-sprocket-readiness-assessment.md | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 reports/mledb-sprocket-readiness-assessment.md diff --git a/reports/mledb-sprocket-readiness-assessment.md b/reports/mledb-sprocket-readiness-assessment.md new file mode 100644 index 000000000..cb892ed5b --- /dev/null +++ b/reports/mledb-sprocket-readiness-assessment.md @@ -0,0 +1,97 @@ +# MLEDB to Sprocket Readiness Assessment + +## Executive Summary + +Sprocket is not ready for a full MLEDB retirement migration. + +The Sprocket schema has usable equivalents for the league-core slice already covered by `queries/migration/mledb-migration.sql`: organization structure, franchises, skill groups, users, members, players, seasons, weeks, fixtures, and matches. `core/` also has services/resolvers/controllers for most of that slice. + +The rest of MLEDB is mixed: + +- Some concepts have Sprocket tables but no backfill or no first-class core service surface. +- Some concepts still depend on legacy `core/src/mledb/*` service code after partial Sprocket writes. +- Some concepts have no clear semantic target in Sprocket at all. + +The practical conclusion is that we can rehearse and validate a league-core migration now, but we cannot migrate all MLEDB data and expect the application to use it without more schema mapping and service work. + +## Readiness Categories + +- `Ready`: Sprocket has a semantic target and core has usable service/resolver/controller paths. +- `Partial`: Sprocket has a likely target, but data fidelity, backfill, or service coverage is incomplete. +- `Legacy-dependent`: Sprocket has some target path, but current core workflows still read/write MLEDB for the behavior. +- `Not ready`: no clear Sprocket semantic target or no usable core path. + +## Table-Level Assessment + +| MLEDB table | Sprocket equivalent | Core service readiness | Status | Evidence | +|---|---|---|---|---| +| `division` | `franchise_group`, `franchise_group_profile`, `franchise_group_type` | Readable through franchise graph population; no dedicated division CRUD service. | Partial | Migration validates name/conference. Core has `FranchiseService`, but no direct franchise-group service surface. | +| `team` | `franchise`, `franchise_profile`, `team`, staff/leadership appointments, roster slots | Franchise/team reads exist; roster/staff sync still consumes MLEDB team rows. | Legacy-dependent | `RosterAuthorityService` injects `MLE_Team`, `MLE_TeamToCaptain`, bridge tables, roster slots, and appointments. | +| `team_branding` | `franchise_profile`, `photo`, webhook fields | Franchise profile exists; logo/photo and webhook mapping are incomplete in migration validation. | Partial | Migration validates code/colors; current validation does not prove photo/logo equivalence. | +| `league_branding` | `game_skill_group_profile`, `photo` | Skill-group services exist; no backfill/validation for color, badge, or emoji from `league_branding`. | Partial | `GameSkillGroupProfile` has `color`, `photo`, and `discordEmojiId`; migration seeds hard-coded profile values instead of using `mledb.league_branding`. | +| `salary_cap` | `game_skill_group.salaryCap` | Skill-group read services exist; no per-row historical salary-cap mapping. | Partial | Sprocket stores one salary cap per skill group, while MLEDB table is league keyed. | +| `player` | `user`, `user_profile`, `user_authentication_account`, `member`, `member_profile`, `player` | Strong service coverage, but player write paths still mirror into MLEDB. | Legacy-dependent | `PlayerService` creates/updates Sprocket rows and also calls `mle_createPlayer`, `mle_updatePlayer`, and MLEDB account mirrors. | +| `player_account` | `member_platform_account` plus `platform` | Sprocket service exists; cutover still falls back to and mirrors MLEDB player accounts. | Legacy-dependent | `MemberPlatformAccountService` exists; `MledbPlayerAccountService` is explicitly documented as a legacy mirror. | +| `player_to_org` | `user_org_team_permission` | Dedicated service and resolution path exist, with dual-read migration behavior. | Partial | Prior org-team dual-read work covers this, but stale legacy grants and flag-off discipline remain required before retirement. | +| `player_history` | no clear equivalent | No Sprocket history/audit model for player profile, team, salary, league, suspension history. | Not ready | Only current-state `player`/`member`/profile models are present. | +| `team_to_captain` | `franchise_staff_appointment`, `roster_slot`, `roster_role` | Sync logic exists, but it is driven from MLEDB captain rows. | Legacy-dependent | `RosterAuthorityService` reads `MLE_TeamToCaptain` to create Sprocket appointments. | +| `draft_order` | `draft_pick`, `draft_selection` | Database entities exist; no core service/resolver/controller outside database module. | Partial | `core/src/database/draft/*` exists, but no `core/src/draft/*` application service exists. | +| `season` | `schedule_group` with type `SEASON` | Service exists and creates both Sprocket and MLEDB rows today. | Legacy-dependent | `ScheduleGroupService` injects `ScheduleGroup`, `MLE_Season`, and bridge rows. | +| `match` | `schedule_group` with type `WEEK` | Service exists and creates both Sprocket and MLEDB rows today. | Legacy-dependent | `ScheduleGroupService` injects `MLE_Match` and `MatchToScheduleGroup`. | +| `fixture` | `schedule_fixture` | Service/resolver exist, but create path still writes MLEDB fixture/series bridge data. | Legacy-dependent | `ScheduleFixtureService` injects `ScheduleFixture`, `MLE_Fixture`, `MLE_Series`, and bridge repos. | +| `series` | `match_parent`, `match`, `scheduled_event` for event-like series | Match service/resolver exist; match operations still bridge to MLEDB series/replays for some mutations. | Legacy-dependent | `MatchResolver` injects `MLE_Series`, `MLE_SeriesReplay`, and `SeriesToMatchParent`. | +| `series_replay` | `round` plus `round.roundStats`, `player_stat_line`, `team_stat_line` | Sprocket finalization writes rounds/stat lines; legacy match/finalization code still uses `MLE_SeriesReplay`. | Legacy-dependent | Rocket League finalization inserts Sprocket `Round`, `TeamStatLine`, and `PlayerStatLine`; MLEDB finalization still inserts MLEDB replays. | +| `player_stats_core` | `player_stat_line.stats` JSON core stats | Finalization writes Sprocket stat lines; no historical backfill/semantic validator exists. | Partial | Sprocket stores JSON stats rather than typed columns; consumers must parse schema. | +| `player_stats` | `player_stat_line.stats` JSON extended stats | Same as above; possible target exists but not typed or validated. | Partial | `PlayerStatLine` has a JSON `stats` field; `PlayerStatLineStatsSchema` covers expected runtime shape only. | +| `team_core_stats` | `team_stat_line.stats` JSON | Same as above; possible target exists but not typed or validated. | Partial | `TeamStatLine` has a JSON `stats` field; no migration equivalence query exists. | +| `team_role_usage` | `roster_role_usage` | Service exists for NCP input, but it writes both MLEDB and Sprocket and skips Sprocket rows when roster mapping is missing. | Legacy-dependent | `MledbNcpTeamRoleUsageService` creates `MLE_TeamRoleUsage` and `RosterRoleUsage`. | +| `scrim` | `scrim_meta`, `match_parent`, scrim module types | Sprocket scrim service exists, but migration of historical MLEDB scrims is not mapped. | Partial | `ScrimMeta` is minimal (`isCompetitive` + parent); MLEDB has mode/type/base points/author/host. | +| `eligibility_data` | `eligibility_data` | Service exists for Sprocket eligibility calculations; no MLEDB backfill mapping yet. | Partial | `EligibilityService` reads Sprocket `EligibilityData`; MLEDB finalization still writes `MLE_EligibilityData`. | +| `elo_data` | no durable Sprocket table; external connector/materialized MLEDB view | Core Elo service still reads `mledb.v_current_elo_values`; connector can calculate external Elo jobs. | Not ready | `EloService` refreshes/selects from MLEDB materialized view and has a TODO to match Sprocket skill group codes. | +| `psyonix_api_result` | no clear equivalent | No Sprocket service or model for historical Psyonix API rank snapshots. | Not ready | Only MLEDB model references were found. | +| `channel_map` | possibly `webhook`, organization config, or profile webhook fields | No direct Sprocket channel-map model or core service. | Not ready | `Webhook` stores URLs, not Discord channel IDs by channel type. | +| `stream_event` | possibly `scheduled_event` | `ScheduledEvent` exists as a model; no service/resolver/controller and no stream-specific mapping. | Partial | Model has description/start/url/host/game/matches, but no application service was found. | +| `config` | `sprocket_configuration` or `organization_configuration_*` | Services exist, but key-by-key semantic mapping from MLEDB config is absent. | Partial | Configuration services support Sprocket config and org config, but MLEDB config keys are not mapped. | +| `footers` | possibly `verbiage` | Verbiage service exists, but footer semantics and codes are not mapped. | Partial | `Verbiage` is code/organization/term; `footers` is free text rows. | + +## Service-Code Findings + +The strongest Sprocket-native areas are: + +- identity/user/member/profile CRUD and GraphQL/RPC surfaces, +- organization/franchise/player/game/skill-group reads and writes, +- schedule group, schedule fixture, match, round, scrim, eligibility, and report-card operational paths, +- Sprocket configuration and organization configuration reads/writes. + +The weakest areas are: + +- draft data: database entities exist but no core application service was found, +- stream events: database entity exists but no core application service was found, +- historical player state: no Sprocket target equivalent was found, +- Elo history/current values: still MLEDB-view-driven from `core/src/elo/elo.service.ts`, +- Psyonix rank snapshots: no Sprocket target equivalent was found, +- typed/statistical historical replay data: Sprocket stores JSON stat lines, but there is no backfill or semantic validator for MLEDB stat columns. + +The major cutover risk is not just schema coverage; it is that several active write paths are dual-write or legacy-driven: + +- player create/update mirrors into `mledb.player` and `mledb.player_account`, +- schedule creation writes `mledb.season`, `mledb.match`, `mledb.fixture`, and `mledb.series`, +- roster authority uses MLEDB team/captain rows as input, +- match mutations still read/write MLEDB series/replay rows, +- NCP role usage writes both MLEDB and Sprocket rows. + +## Blocking Gaps Before Full Migration + +1. Define authoritative target mappings for every `Partial`, `Legacy-dependent`, and `Not ready` row above. +2. Decide whether historical-only MLEDB tables should be migrated to Sprocket, archived read-only, or intentionally dropped. +3. Replace legacy-driven write paths with Sprocket-authoritative services before disabling MLEDB writes. +4. Add backfills and semantic validators for stats, scrims, draft order, player accounts, role usage, staff/captain roles, eligibility, configuration, webhooks/channel routing, and stream/scheduled events. +5. Build a post-migration runtime smoke plan that exercises the Sprocket-native service paths without MLEDB fallback. + +## Recommended Sequence + +1. Treat the current PR as a league-core rehearsal tool, not a full-MLEDB retirement tool. +2. Add one migration slice at a time for deferred domains, starting with platform accounts, roster/staff/captains, and role usage because active core workflows already straddle those domains. +3. Move schedule/match/player write paths to Sprocket-authoritative behavior behind flags, then validate they no longer require MLEDB writes. +4. Decide the retention strategy for player history, Elo history, Psyonix snapshots, and full replay/stat history before attempting bulk migration. +5. Only after the above, run the guardrail workflow against a production dump and require zero semantic issues plus zero changed/deleted pre-existing Sprocket rows.