From c3432a7da0661844445341a0d4b8a13b48f088a1 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 31 May 2026 13:12:55 -0700 Subject: [PATCH] fix(schema): widen MusicBrainz Discogs ID columns to BIGINT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit brainztableinator throws continuous PostgreSQL "integer out of range" errors when processing MusicBrainz releases: discogs_release_id (and its siblings discogs_artist_id, discogs_label_id, discogs_master_id) were declared INTEGER (int4, max 2,147,483,647), but the extractor parses Discogs IDs as i64 and release IDs now exceed 2.1B. Fix both fresh installs and existing databases: - Change the four cross-reference columns to BIGINT in the CREATE TABLE definitions (fresh installs). - Add idempotent `ALTER COLUMN ... TYPE BIGINT` migrations, since CREATE TABLE IF NOT EXISTS is a no-op on pre-existing tables. Neo4j (brainzgraphinator) is unaffected — Neo4j integers are 64-bit. Co-Authored-By: Claude Opus 4.8 (1M context) --- brainztableinator/README.md | 8 ++--- schema-init/postgres_schema.py | 29 +++++++++++++++--- tests/schema-init/test_musicbrainz_schema.py | 31 +++++++++++++++++++- tests/schema-init/test_postgres_schema.py | 4 +++ 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/brainztableinator/README.md b/brainztableinator/README.md index f1ae4c40..4ea2086d 100644 --- a/brainztableinator/README.md +++ b/brainztableinator/README.md @@ -80,7 +80,7 @@ CREATE TABLE musicbrainz.artists ( begin_area TEXT, end_area TEXT, disambiguation TEXT, - discogs_artist_id INTEGER, + discogs_artist_id BIGINT, aliases JSONB, tags JSONB, data JSONB, @@ -102,7 +102,7 @@ CREATE TABLE musicbrainz.labels ( ended BOOLEAN DEFAULT FALSE, area TEXT, disambiguation TEXT, - discogs_label_id INTEGER, + discogs_label_id BIGINT, data JSONB, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() @@ -118,7 +118,7 @@ CREATE TABLE musicbrainz.releases ( barcode TEXT, status TEXT, release_group_mbid UUID, - discogs_release_id INTEGER, + discogs_release_id BIGINT, data JSONB, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() @@ -135,7 +135,7 @@ CREATE TABLE musicbrainz.release_groups ( secondary_types JSONB, first_release_date TEXT, disambiguation TEXT, - discogs_master_id INTEGER, + discogs_master_id BIGINT, data JSONB, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() diff --git a/schema-init/postgres_schema.py b/schema-init/postgres_schema.py index 853a172a..9f1074c6 100644 --- a/schema-init/postgres_schema.py +++ b/schema-init/postgres_schema.py @@ -539,7 +539,7 @@ begin_area TEXT, end_area TEXT, disambiguation TEXT, - discogs_artist_id INTEGER, + discogs_artist_id BIGINT, aliases JSONB, tags JSONB, data JSONB, @@ -559,7 +559,7 @@ ended BOOLEAN DEFAULT FALSE, area TEXT, disambiguation TEXT, - discogs_label_id INTEGER, + discogs_label_id BIGINT, data JSONB, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() @@ -573,7 +573,7 @@ barcode TEXT, status TEXT, release_group_mbid UUID, - discogs_release_id INTEGER, + discogs_release_id BIGINT, data JSONB, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() @@ -588,7 +588,7 @@ secondary_types JSONB, first_release_date TEXT, disambiguation TEXT, - discogs_master_id INTEGER, + discogs_master_id BIGINT, data JSONB, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW() @@ -623,6 +623,27 @@ UNIQUE (mbid, entity_type, service_name, url) )""", ), + # Widen Discogs cross-reference IDs from INTEGER (int4, max 2,147,483,647) + # to BIGINT. Discogs IDs are emitted as i64 by the extractor and release IDs + # now exceed 2.1B, causing "integer out of range" on INSERT. CREATE TABLE + # IF NOT EXISTS above won't alter pre-existing tables, so widen explicitly. + # ALTER COLUMN ... TYPE BIGINT is a no-op when the column is already BIGINT. + ( + "musicbrainz.artists.discogs_artist_id widen to BIGINT", + "ALTER TABLE musicbrainz.artists ALTER COLUMN discogs_artist_id TYPE BIGINT", + ), + ( + "musicbrainz.labels.discogs_label_id widen to BIGINT", + "ALTER TABLE musicbrainz.labels ALTER COLUMN discogs_label_id TYPE BIGINT", + ), + ( + "musicbrainz.releases.discogs_release_id widen to BIGINT", + "ALTER TABLE musicbrainz.releases ALTER COLUMN discogs_release_id TYPE BIGINT", + ), + ( + "musicbrainz.release_groups.discogs_master_id widen to BIGINT", + "ALTER TABLE musicbrainz.release_groups ALTER COLUMN discogs_master_id TYPE BIGINT", + ), ] diff --git a/tests/schema-init/test_musicbrainz_schema.py b/tests/schema-init/test_musicbrainz_schema.py index ad6c6889..a359538c 100644 --- a/tests/schema-init/test_musicbrainz_schema.py +++ b/tests/schema-init/test_musicbrainz_schema.py @@ -14,11 +14,40 @@ def test_musicbrainz_tables_defined(): def test_musicbrainz_tables_use_if_not_exists(): + # ALTER migrations (e.g. widening columns to BIGINT) cannot use IF NOT EXISTS; + # only CREATE TABLE statements must be idempotent via IF NOT EXISTS. for name, sql in _MUSICBRAINZ_TABLES: - if name != "musicbrainz schema": + if "CREATE TABLE" in sql: assert "IF NOT EXISTS" in sql, f"{name} missing IF NOT EXISTS" +def test_musicbrainz_discogs_id_columns_are_bigint(): + # Discogs IDs are i64 from the extractor and now exceed int4 range (2.1B); + # cross-reference columns must be BIGINT to avoid "integer out of range". + expected = { + "musicbrainz.artists table": "discogs_artist_id BIGINT", + "musicbrainz.labels table": "discogs_label_id BIGINT", + "musicbrainz.releases table": "discogs_release_id BIGINT", + "musicbrainz.release_groups table": "discogs_master_id BIGINT", + } + found = dict.fromkeys(expected, False) + for name, sql in _MUSICBRAINZ_TABLES: + if name in expected: + assert expected[name] in sql, f"{name} missing {expected[name]}" + found[name] = True + assert all(found.values()), f"missing tables: {[n for n, ok in found.items() if not ok]}" + + +def test_musicbrainz_has_bigint_widening_migrations(): + # Existing databases need explicit ALTER COLUMN ... TYPE BIGINT; CREATE TABLE + # IF NOT EXISTS is a no-op on tables that already exist as INTEGER. + migrations = [sql for name, sql in _MUSICBRAINZ_TABLES if "ALTER COLUMN" in sql] + assert any("musicbrainz.artists ALTER COLUMN discogs_artist_id TYPE BIGINT" in m for m in migrations) + assert any("musicbrainz.labels ALTER COLUMN discogs_label_id TYPE BIGINT" in m for m in migrations) + assert any("musicbrainz.releases ALTER COLUMN discogs_release_id TYPE BIGINT" in m for m in migrations) + assert any("musicbrainz.release_groups ALTER COLUMN discogs_master_id TYPE BIGINT" in m for m in migrations) + + def test_musicbrainz_indexes_defined(): index_names = [name for name, _ in _MUSICBRAINZ_INDEXES] assert "idx_mb_artists_discogs_id" in index_names diff --git a/tests/schema-init/test_postgres_schema.py b/tests/schema-init/test_postgres_schema.py index 9229f412..5d3d5763 100644 --- a/tests/schema-init/test_postgres_schema.py +++ b/tests/schema-init/test_postgres_schema.py @@ -232,6 +232,10 @@ async def capture(stmt: Any, *_: Any, **__: Any) -> None: for stmt in captured: upper = stmt.upper() + # ALTER COLUMN ... TYPE is inherently idempotent (re-applying the same + # target type is a no-op), so it needs no IF NOT EXISTS guard. + if "ALTER COLUMN" in upper and "TYPE " in upper: + continue assert "IF NOT EXISTS" in upper, f"Statement is not idempotent: {stmt[:80]}..." # A multi-statement blob could still hide an un-guarded DROP that would # not be idempotent — any DROP must be guarded with IF EXISTS.